| Message ID | 20240209152547.2741703-1-thomas.perrot@bootlin.com |
|---|---|
| State | New |
| Headers | show |
| Series | [bitbake-devel] wget.py: always use the custom user agent | expand |
Thomas Perrot via lists.openembedded.org <thomas.perrot= bootlin.com@lists.openembedded.org> escreveu (sexta, 9/02/2024 à(s) 15:25): > From: Thomas Perrot <thomas.perrot@bootlin.com> > > Add the "--user-agent" paramater in the wget base command to > perform all wget commands with this parameter, because a few > HTTP servers block requests with the default wget user agent. > > For example, "hg.openjdk.org" never send a response to requests > have been sent with wget: > wget https://hg.openjdk.org/jdk8u/jdk8u/archive/jdk8u272-ga.tar.bz2 > https://hg.openjdk.org/jdk8u/jdk8u/archive/jdk8u272-ga.tar.bz2 > Resolving hg.openjdk.org (hg.openjdk.org)... 23.54.129.73 > Connecting to hg.openjdk.org (hg.openjdk.org)|23.54.129.73|:443... > connected. > HTTP request sent, awaiting response... > > Signed-off-by: Thomas Perrot <thomas.perrot@bootlin.com> > --- > lib/bb/fetch2/wget.py | 7 ++++--- > 1 file changed, 4 insertions(+), 3 deletions(-) > > diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py > index dc025800e659..e7c4cdd396b3 100644 > --- a/lib/bb/fetch2/wget.py > +++ b/lib/bb/fetch2/wget.py > @@ -87,7 +87,8 @@ class Wget(FetchMethod): > if not ud.localfile: > ud.localfile = d.expand(urllib.parse.unquote(ud.host + > ud.path).replace("/", ".")) > > - self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t > 2 -T 30 --passive-ftp" > + self.basecmd = d.getVar("FETCHCMD_wget") \ > + or "/usr/bin/env wget -t 2 -T 30 --passive-ftp > --user-agent='%s'" % (self.user_agent) > > if not self.check_certs(d): > self.basecmd += " --no-check-certificate" > @@ -454,7 +455,7 @@ class Wget(FetchMethod): > f = tempfile.NamedTemporaryFile() > with tempfile.TemporaryDirectory(prefix="wget-index-") as > workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") > as f: > fetchcmd = self.basecmd > - fetchcmd += " -O " + f.name + " --user-agent='" + > self.user_agent + "' '" + uri + "'" > + fetchcmd += " -O '" + uri + "'" > Looks like the file name was removed unintentional. Jose > try: > self._runwget(ud, d, fetchcmd, True, workdir=workdir) > fetchresult = f.read() > @@ -492,7 +493,7 @@ class Wget(FetchMethod): > valid = 1 > elif self._vercmp(version, newver) < 0: > version = newver > - > + > pupver = re.sub('_', '.', version[1]) > > bb.debug(3, "*** %s -> UpstreamVersion = %s (CurrentVersion = > %s)" % > -- > 2.43.0 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#15851): > https://lists.openembedded.org/g/bitbake-devel/message/15851 > Mute This Topic: https://lists.openembedded.org/mt/104260370/5052612 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ > quaresma.jose@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
Hello Jose, On Fri, 2024-02-09 at 15:52 +0000, Jose Quaresma wrote: > > > Thomas Perrot via lists.openembedded.org > <thomas.perrot=bootlin.com@lists.openembedded.org> escreveu (sexta, > 9/02/2024 à(s) 15:25): > > From: Thomas Perrot <thomas.perrot@bootlin.com> > > > > Add the "--user-agent" paramater in the wget base command to > > perform all wget commands with this parameter, because a few > > HTTP servers block requests with the default wget user agent. > > > > For example, "hg.openjdk.org" never send a response to requests > > have been sent with wget: > > wget https://hg.openjdk.org/jdk8u/jdk8u/archive/jdk8u272-ga.tar.bz2 > > https://hg.openjdk.org/jdk8u/jdk8u/archive/jdk8u272-ga.tar.bz2 > > Resolving hg.openjdk.org (hg.openjdk.org)... 23.54.129.73 > > Connecting to hg.openjdk.org (hg.openjdk.org)|23.54.129.73|:443... > > connected. > > HTTP request sent, awaiting response... > > > > Signed-off-by: Thomas Perrot <thomas.perrot@bootlin.com> > > --- > > lib/bb/fetch2/wget.py | 7 ++++--- > > 1 file changed, 4 insertions(+), 3 deletions(-) > > > > diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py > > index dc025800e659..e7c4cdd396b3 100644 > > --- a/lib/bb/fetch2/wget.py > > +++ b/lib/bb/fetch2/wget.py > > @@ -87,7 +87,8 @@ class Wget(FetchMethod): > > if not ud.localfile: > > ud.localfile = d.expand(urllib.parse.unquote(ud.host + > > ud.path).replace("/", ".")) > > > > - self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env > > wget -t 2 -T 30 --passive-ftp" > > + self.basecmd = d.getVar("FETCHCMD_wget") \ > > + or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --user- > > agent='%s'" % (self.user_agent) > > > > if not self.check_certs(d): > > self.basecmd += " --no-check-certificate" > > @@ -454,7 +455,7 @@ class Wget(FetchMethod): > > f = tempfile.NamedTemporaryFile() > > with tempfile.TemporaryDirectory(prefix="wget-index-") as > > workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget- > > listing-") as f: > > fetchcmd = self.basecmd > > - fetchcmd += " -O " + f.name + " --user-agent='" + > > self.user_agent + "' '" + uri + "'" > > + fetchcmd += " -O '" + uri + "'" > > > > > Looks like the file name was removed unintentional. Really sorry for this error, I'm sending a v2. Kind regards, Thomas Perrot > > Jose > > > try: > > self._runwget(ud, d, fetchcmd, True, > > workdir=workdir) > > fetchresult = f.read() > > @@ -492,7 +493,7 @@ class Wget(FetchMethod): > > valid = 1 > > elif self._vercmp(version, newver) < 0: > > version = newver > > - > > + > > pupver = re.sub('_', '.', version[1]) > > > > bb.debug(3, "*** %s -> UpstreamVersion = %s > > (CurrentVersion = %s)" % > > > > -=-=-=-=-=-=-=-=-=-=-=- > > Links: You receive all messages sent to this group. > > View/Reply Online (#15853): > > https://lists.openembedded.org/g/bitbake-devel/message/15853 > > Mute This Topic: > > https://lists.openembedded.org/mt/104260370/5443093 > > Group Owner: bitbake-devel+owner@lists.openembedded.org > > Unsubscribe: > > https://lists.openembedded.org/g/bitbake-devel/unsub [thomas.perrot@bootlin.com > > ] > > -=-=-=-=-=-=-=-=-=-=-=- > >
diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py index dc025800e659..e7c4cdd396b3 100644 --- a/lib/bb/fetch2/wget.py +++ b/lib/bb/fetch2/wget.py @@ -87,7 +87,8 @@ class Wget(FetchMethod): if not ud.localfile: ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", ".")) - self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp" + self.basecmd = d.getVar("FETCHCMD_wget") \ + or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --user-agent='%s'" % (self.user_agent) if not self.check_certs(d): self.basecmd += " --no-check-certificate" @@ -454,7 +455,7 @@ class Wget(FetchMethod): f = tempfile.NamedTemporaryFile() with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f: fetchcmd = self.basecmd - fetchcmd += " -O " + f.name + " --user-agent='" + self.user_agent + "' '" + uri + "'" + fetchcmd += " -O '" + uri + "'" try: self._runwget(ud, d, fetchcmd, True, workdir=workdir) fetchresult = f.read() @@ -492,7 +493,7 @@ class Wget(FetchMethod): valid = 1 elif self._vercmp(version, newver) < 0: version = newver - + pupver = re.sub('_', '.', version[1]) bb.debug(3, "*** %s -> UpstreamVersion = %s (CurrentVersion = %s)" %