From dcf9bf3cf137f9365a9f85bed7262894138dbfaa Mon Sep 17 00:00:00 2001 From: Michal Domonkos Date: Tue, 21 May 2019 17:42:49 +0200 Subject: [PATCH 1/2] Start with Patch1 instead of 0 More consistent with the actual filenames produced by "git format-patch". --- python-urlgrabber.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-urlgrabber.spec b/python-urlgrabber.spec index 00a2191..dd99d71 100644 --- a/python-urlgrabber.spec +++ b/python-urlgrabber.spec @@ -21,7 +21,7 @@ URL: http://urlgrabber.baseurl.org/ #Source0: http://urlgrabber.baseurl.org/download/urlgrabber-%{version}.tar.gz Source0: https://github.com/rpm-software-management/urlgrabber/releases/download/urlgrabber-%{dashversion}/urlgrabber-%{version}.tar.gz -Patch0: 0001-urlgrabber-ext-down-another-python-3-compat.patch +Patch1: 0001-urlgrabber-ext-down-another-python-3-compat.patch BuildArch: noarch From a6f3cf248deda868a8707e5faaeb68575e2ab157 Mon Sep 17 00:00:00 2001 From: Michal Domonkos Date: Tue, 21 May 2019 18:06:23 +0200 Subject: [PATCH 2/2] Fix mirror parsing regression (rhbz#1710576) python3: fix urlgrabber-ext-down to pass correct options to pycurl Fix failing unit test covering the retry_no_cache option Resolves: rhbz#1710576 --- ...t-Simplify-mirror-conversion-to-utf8.patch | 40 +++++++++++++ ...bber-ext-down-convert-url-into-bytes.patch | 56 +++++++++++++++++++ ...own-file-content-in-test_retry_no_ca.patch | 34 +++++++++++ python-urlgrabber.spec | 10 +++- 4 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 0002-Revert-Simplify-mirror-conversion-to-utf8.patch create mode 100644 0003-urlgrabber-ext-down-convert-url-into-bytes.patch create mode 100644 0004-test-handle-unknown-file-content-in-test_retry_no_ca.patch diff --git a/0002-Revert-Simplify-mirror-conversion-to-utf8.patch b/0002-Revert-Simplify-mirror-conversion-to-utf8.patch new file mode 100644 index 0000000..464c14d --- /dev/null +++ b/0002-Revert-Simplify-mirror-conversion-to-utf8.patch @@ -0,0 +1,40 @@ +From 7607b9f408b71b6533ca4f8e8808090a5b930555 Mon Sep 17 00:00:00 2001 +From: Michal Domonkos +Date: Mon, 20 May 2019 15:06:38 +0200 +Subject: [PATCH 2/4] Revert "Simplify mirror conversion to utf8" + +This reverts commit be8ee10e35319e80200d4ff384434d46fe7783d9. + +A list of dicts (as opposed to strings) is valid input as well; see the +module-level doc string for details (section 2 under CUSTOMIZATION). In +fact, the nested estimate() function in MirrorGroup.__init__() accounts +for that, too. + +This fixes a traceback in YUM which does pass such a dict list. + +Closes #10. +--- + urlgrabber/mirror.py | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py +index 75f0bcb..d95863e 100644 +--- a/urlgrabber/mirror.py ++++ b/urlgrabber/mirror.py +@@ -297,7 +297,12 @@ class MirrorGroup: + self.default_action = kwargs.get('default_action') + + def _parse_mirrors(self, mirrors): +- return [{'mirror':_to_utf8(m)} for m in mirrors] ++ parsed_mirrors = [] ++ for m in mirrors: ++ if isinstance(m, string_types): ++ m = {'mirror': _to_utf8(m)} ++ parsed_mirrors.append(m) ++ return parsed_mirrors + + def _load_gr(self, gr): + # OVERRIDE IDEAS: +-- +2.21.0 + diff --git a/0003-urlgrabber-ext-down-convert-url-into-bytes.patch b/0003-urlgrabber-ext-down-convert-url-into-bytes.patch new file mode 100644 index 0000000..67eff79 --- /dev/null +++ b/0003-urlgrabber-ext-down-convert-url-into-bytes.patch @@ -0,0 +1,56 @@ +From c696255c5f50341688d718b3547efb1a9971927b Mon Sep 17 00:00:00 2001 +From: Michal Domonkos +Date: Tue, 21 May 2019 11:12:12 +0200 +Subject: [PATCH 3/4] urlgrabber-ext-down: convert url into bytes + +We need to convert the parsed url back into bytes before passing it to +the PyCurlFileObject constructor (since _set_opts() expects self.scheme, +constructed from the url, to be a bytes object). + +This caused the unit test "bypassing proxy cache on failure" to fail +(together with a bug in the test itself which is also being fixed here). + +Closes #14. +--- + scripts/urlgrabber-ext-down | 4 ++-- + test/test_mirror.py | 2 +- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/scripts/urlgrabber-ext-down b/scripts/urlgrabber-ext-down +index 13d6dc7..40469a7 100755 +--- a/scripts/urlgrabber-ext-down ++++ b/scripts/urlgrabber-ext-down +@@ -22,7 +22,7 @@ import time, os, errno, sys + import six + from urlgrabber.grabber import \ + _readlines, URLGrabberOptions, _loads, \ +- PyCurlFileObject, URLGrabError ++ PyCurlFileObject, URLGrabError, _to_utf8 + + def write(fmt, *arg): + buf = fmt % arg +@@ -65,7 +65,7 @@ def main(): + + dlsz = dltm = 0 + try: +- fo = PyCurlFileObject(opts.url, opts.filename, opts) ++ fo = PyCurlFileObject(_to_utf8(opts.url), opts.filename, opts) + fo._do_grab() + fo.fo.close() + size = fo._amount_read +diff --git a/test/test_mirror.py b/test/test_mirror.py +index a2daf19..66cafd0 100644 +--- a/test/test_mirror.py ++++ b/test/test_mirror.py +@@ -350,7 +350,7 @@ class HttpReplyCode(TestCase): + self.content = b'version1' + + def checkfunc_read(obj): +- if obj.data == 'version1': ++ if obj.data == b'version1': + raise URLGrabError(-1, 'Outdated version of foo') + + def checkfunc_grab(obj): +-- +2.21.0 + diff --git a/0004-test-handle-unknown-file-content-in-test_retry_no_ca.patch b/0004-test-handle-unknown-file-content-in-test_retry_no_ca.patch new file mode 100644 index 0000000..3f7f5ee --- /dev/null +++ b/0004-test-handle-unknown-file-content-in-test_retry_no_ca.patch @@ -0,0 +1,34 @@ +From 2bdf588a20ee9d9175fb27a819d479284b7e5079 Mon Sep 17 00:00:00 2001 +From: Michal Domonkos +Date: Tue, 21 May 2019 14:36:36 +0200 +Subject: [PATCH 4/4] test: handle unknown file content in test_retry_no_cache + +--- + test/test_mirror.py | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/test/test_mirror.py b/test/test_mirror.py +index 66cafd0..a175977 100644 +--- a/test/test_mirror.py ++++ b/test/test_mirror.py +@@ -352,11 +352,16 @@ class HttpReplyCode(TestCase): + def checkfunc_read(obj): + if obj.data == b'version1': + raise URLGrabError(-1, 'Outdated version of foo') ++ elif obj.data != b'version2': ++ self.fail('Unexpected file content') + + def checkfunc_grab(obj): + with open('foo') as f: +- if f.read() == 'version1': ++ data = f.read() ++ if data == 'version1': + raise URLGrabError(-1, 'Outdated version of foo') ++ elif data != 'version2': ++ self.fail('Unexpected file content') + + self.process = process + self.reply = 200, b'OK' +-- +2.21.0 + diff --git a/python-urlgrabber.spec b/python-urlgrabber.spec index dd99d71..e57adae 100644 --- a/python-urlgrabber.spec +++ b/python-urlgrabber.spec @@ -11,7 +11,7 @@ Name: python-%{pypi_name} Version: %{majorver}.%{minorver}.%{patchver} -Release: 2%{?dist} +Release: 3%{?dist} Summary: A high-level cross-protocol url-grabber License: LGPLv2+ @@ -22,6 +22,9 @@ URL: http://urlgrabber.baseurl.org/ Source0: https://github.com/rpm-software-management/urlgrabber/releases/download/urlgrabber-%{dashversion}/urlgrabber-%{version}.tar.gz Patch1: 0001-urlgrabber-ext-down-another-python-3-compat.patch +Patch2: 0002-Revert-Simplify-mirror-conversion-to-utf8.patch +Patch3: 0003-urlgrabber-ext-down-convert-url-into-bytes.patch +Patch4: 0004-test-handle-unknown-file-content-in-test_retry_no_ca.patch BuildArch: noarch @@ -99,6 +102,11 @@ export URLGRABBER_EXT_DOWN="%{_builddir}/%{pypi_name}-%{version}/scripts/urlgrab %{python3_sitelib}/%{pypi_name}-%{version}-py?.?.egg-info %changelog +* Tue May 21 2019 Michal Domonkos - 4.0.0-3 +- Fix mirror parsing regression (rhbz#1710576) +- python3: fix urlgrabber-ext-down to pass correct options to pycurl +- Fix failing unit test covering the retry_no_cache option + * Sun May 12 2019 Pavel Raiskup - 4.0.0-2 - python3 compat work-around to fix mock (rhbz#1707657, rhbz#1688173)