rpms
/
python-urlgrabber
20
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert porting to Python 3.

Browse Source
epel9
Valentina Mukhamedzhanova 10 years ago
parent b791d6f370
commit 2839b3e6c4
4 changed files with 6 additions and 2275 deletions
Show Stats Download Patch File Download Diff File

658
port-tests-to-python3.patch
Unescape View File

@ -1,658 +0,0 @@
commit 8560a386c3ea1e868a8e294c1e318a6ee5319580
Author: Tomas Radej <tradej@redhat.com>
Date: Wed Aug 20 13:32:32 2014 +0200
Ported test suite
diff --git a/test/grabberperf.py b/test/grabberperf.py
index 820da2c..d9142fa 100644
--- a/test/grabberperf.py
+++ b/test/grabberperf.py
@@ -21,11 +21,15 @@
import sys
import os
-from os.path import dirname, join as joinpath
import tempfile
import time
+import six
+
+# Hack for Python 3
+sys.path.insert(0, os.path.expandvars(os.path.abspath('..')))
-import urlgrabber.grabber as grabber
+from os.path import dirname, join as joinpath
+from urlgrabber import grabber
from urlgrabber.grabber import URLGrabber, urlgrab, urlopen, urlread
from urlgrabber.progress import text_progress_meter
@@ -48,7 +52,7 @@ def main():
os.unlink(tempdst)
def setuptemp(size):
- if DEBUG: print 'writing %d KB to temporary file (%s).' % (size / 1024, tempsrc)
+ if DEBUG: print('writing %d KB to temporary file (%s).' % (size / 1024, tempsrc))
file = open(tempsrc, 'w', 1024)
chars = '0123456789'
for i in range(size):
@@ -65,9 +69,9 @@ def speedtest(size):
try:
from urlgrabber.progress import text_progress_meter
- except ImportError, e:
+ except ImportError as e:
tpm = None
- print 'not using progress meter'
+ print('not using progress meter')
else:
tpm = text_progress_meter(fo=open('/dev/null', 'w'))
@@ -83,15 +87,15 @@ def speedtest(size):
# module.
# get it nicely cached before we start comparing
- if DEBUG: print 'pre-caching'
+ if DEBUG: print('pre-caching')
for i in range(100):
urlgrab(tempsrc, tempdst, copy_local=1, throttle=None, proxies=proxies)
- if DEBUG: print 'running speed test.'
+ if DEBUG: print('running speed test.')
reps = 500
for i in range(reps):
if DEBUG:
- print '\r%4i/%-4i' % (i+1, reps),
+ six.print_('\r%4i/%-4i' % (i+1, reps), end=' ')
sys.stdout.flush()
t = time.time()
urlgrab(tempsrc, tempdst,
@@ -111,14 +115,14 @@ def speedtest(size):
while 1:
s = in_fo.read(1024 * 8)
if not s: break
- out_fo.write(s)
+ out_fo.write(s if not six.PY3 else s.encode('utf-8'))
in_fo.close()
out_fo.close()
none_times.append(1000 * (time.time() - t))
- if DEBUG: print '\r'
+ if DEBUG: print('\r')
- print "%d KB Results:" % (size / 1024)
+ print("%d KB Results:" % (size / 1024))
print_result('full', full_times)
print_result('raw', raw_times)
print_result('none', none_times)
@@ -131,7 +135,7 @@ def print_result(label, result_list):
for i in result_list: mean += i
mean = mean/len(result_list)
median = result_list[int(len(result_list)/2)]
- print format % (label, mean, median, result_list[0], result_list[-1])
+ print(format % (label, mean, median, result_list[0], result_list[-1]))
if __name__ == '__main__':
main()
diff --git a/test/munittest.py b/test/munittest.py
index 16a61ae..7e7969e 100644
--- a/test/munittest.py
+++ b/test/munittest.py
@@ -103,9 +103,9 @@ SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
import time
import sys
import traceback
-import string
import os
import types
+import collections
##############################################################################
# Exported classes and functions
@@ -190,7 +190,7 @@ class TestResult:
def _exc_info_to_string(self, err):
"""Converts a sys.exc_info()-style tuple of values into a string."""
- return string.join(traceback.format_exception(*err), '')
+ return ''.join(traceback.format_exception(*err))
def __repr__(self):
return "<%s run=%i errors=%i failures=%i>" % \
@@ -251,8 +251,8 @@ class TestCase:
testMethod = getattr(self, methodName)
self._testMethodDoc = testMethod.__doc__
except AttributeError:
- raise ValueError, "no such test method in %s: %s" % \
- (self.__class__, methodName)
+ raise ValueError("no such test method in %s: %s" % \
+ (self.__class__, methodName))
def setUp(self):
"Hook method for setting up the test fixture before exercising it."
@@ -276,7 +276,7 @@ class TestCase:
the specified test method's docstring.
"""
doc = self._testMethodDoc
- return doc and string.strip(string.split(doc, "\n")[0]) or None
+ return doc and doc.split("\n")[0].strip() or None
def id(self):
return "%s.%s" % (_strclass(self.__class__), self._testMethodName)
@@ -361,15 +361,15 @@ class TestCase:
def fail(self, msg=None):
"""Fail immediately, with the given message."""
- raise self.failureException, msg
+ raise self.failureException(msg)
def failIf(self, expr, msg=None):
"Fail the test if the expression is true."
- if expr: raise self.failureException, msg
+ if expr: raise self.failureException(msg)
def failUnless(self, expr, msg=None):
"""Fail the test unless the expression is true."""
- if not expr: raise self.failureException, msg
+ if not expr: raise self.failureException(msg)
def failUnlessRaises(self, excClass, callableObj, *args, **kwargs):
"""Fail unless an exception of class excClass is thrown
@@ -386,23 +386,21 @@ class TestCase:
else:
if hasattr(excClass,'__name__'): excName = excClass.__name__
else: excName = str(excClass)
- raise self.failureException, excName
+ raise self.failureException(excName)
def failUnlessEqual(self, first, second, msg=None):
"""Fail if the two objects are unequal as determined by the '=='
operator.
"""
if not first == second:
- raise self.failureException, \
- (msg or '%s != %s' % (`first`, `second`))
+ raise self.failureException(msg or '%s != %s' % (repr(first), repr(second)))
def failIfEqual(self, first, second, msg=None):
"""Fail if the two objects are equal as determined by the '=='
operator.
"""
if first == second:
- raise self.failureException, \
- (msg or '%s == %s' % (`first`, `second`))
+ raise self.failureException(msg or '%s == %s' % (repr(first), repr(second)))
def failUnlessAlmostEqual(self, first, second, places=7, msg=None):
"""Fail if the two objects are unequal as determined by their
@@ -413,8 +411,7 @@ class TestCase:
as significant digits (measured from the most significant digit).
"""
if round(second-first, places) != 0:
- raise self.failureException, \
- (msg or '%s != %s within %s places' % (`first`, `second`, `places` ))
+ raise self.failureException(msg or '%s != %s within %s places' % (repr(first), repr(second), repr(places) ))
def failIfAlmostEqual(self, first, second, places=7, msg=None):
"""Fail if the two objects are equal as determined by their
@@ -425,8 +422,7 @@ class TestCase:
as significant digits (measured from the most significant digit).
"""
if round(second-first, places) == 0:
- raise self.failureException, \
- (msg or '%s == %s within %s places' % (`first`, `second`, `places`))
+ raise self.failureException(msg or '%s == %s within %s places' % (repr(first), repr(second), repr(places)))
assertEqual = assertEquals = failUnlessEqual
@@ -442,15 +438,15 @@ class TestCase:
def skip(self, msg=None):
"""Skip the test"""
- raise self.skipException, msg
+ raise self.skipException(msg)
def skipIf(self, expr, msg=None):
"Skip the test if the expression is true."
- if expr: raise self.skipException, msg
+ if expr: raise self.skipException(msg)
def skipUnless(self, expr, msg=None):
"""Skip the test unless the expression is true."""
- if not expr: raise self.skipException, msg
+ if not expr: raise self.skipException(msg)
@@ -554,7 +550,7 @@ class FunctionTestCase(TestCase):
def shortDescription(self):
if self._description is not None: return self._description
doc = self._testFunc.__doc__
- return doc and string.strip(string.split(doc, "\n")[0]) or None
+ return doc and doc.split("\n")[0].strip() or None
@@ -567,13 +563,12 @@ class TestLoader:
criteria and returning them wrapped in a Test
"""
testMethodPrefix = 'test'
- sortTestMethodsUsing = cmp
suiteClass = TestSuite
def loadTestsFromTestCase(self, testCaseClass):
"""Return a suite of all tests cases contained in testCaseClass"""
name_list = self.getTestCaseNames(testCaseClass)
- instance_list = map(testCaseClass, name_list)
+ instance_list = list(map(testCaseClass, name_list))
description = getattr(testCaseClass, '__doc__') \
or testCaseClass.__name__
description = (description.splitlines()[0]).strip()
@@ -585,7 +580,7 @@ class TestLoader:
tests = []
for name in dir(module):
obj = getattr(module, name)
- if (isinstance(obj, (type, types.ClassType)) and
+ if (isinstance(obj, type) and
issubclass(obj, TestCase) and
not obj in [TestCase, FunctionTestCase]):
tests.append(self.loadTestsFromTestCase(obj))
@@ -603,15 +598,15 @@ class TestLoader:
The method optionally resolves the names relative to a given module.
"""
- parts = string.split(name, '.')
+ parts = name.split('.')
if module is None:
if not parts:
- raise ValueError, "incomplete test name: %s" % name
+ raise ValueError("incomplete test name: %s" % name)
else:
parts_copy = parts[:]
while parts_copy:
try:
- module = __import__(string.join(parts_copy,'.'))
+ module = __import__('.'.join(parts_copy,))
break
except ImportError:
del parts_copy[-1]
@@ -624,20 +619,19 @@ class TestLoader:
import unittest
if type(obj) == types.ModuleType:
return self.loadTestsFromModule(obj)
- elif (isinstance(obj, (type, types.ClassType)) and
+ elif (isinstance(obj, type) and
issubclass(obj, unittest.TestCase)):
return self.loadTestsFromTestCase(obj)
elif type(obj) == types.UnboundMethodType:
- return obj.im_class(obj.__name__)
- elif callable(obj):
+ return obj.__self__.__class__(obj.__name__)
+ elif isinstance(obj, collections.Callable):
test = obj()
if not isinstance(test, unittest.TestCase) and \
not isinstance(test, unittest.TestSuite):
- raise ValueError, \
- "calling %s returned %s, not a test" % (obj,test)
+ raise ValueError("calling %s returned %s, not a test" % (obj,test))
return test
else:
- raise ValueError, "don't know how to make test from: %s" % obj
+ raise ValueError("don't know how to make test from: %s" % obj)
def loadTestsFromNames(self, names, module=None):
"""Return a suite of all tests cases found using the given sequence
@@ -651,14 +645,13 @@ class TestLoader:
def getTestCaseNames(self, testCaseClass):
"""Return a sorted sequence of method names found within testCaseClass
"""
- testFnNames = filter(lambda n,p=self.testMethodPrefix: n[:len(p)] == p,
- dir(testCaseClass))
+ testFnNames = list(filter(lambda n,p=self.testMethodPrefix: n[:len(p)] == p,
+ dir(testCaseClass)))
for baseclass in testCaseClass.__bases__:
for testFnName in self.getTestCaseNames(baseclass):
if testFnName not in testFnNames: # handle overridden methods
testFnNames.append(testFnName)
- if self.sortTestMethodsUsing:
- testFnNames.sort(self.sortTestMethodsUsing)
+ testFnNames.sort()
return testFnNames
@@ -670,21 +663,20 @@ defaultTestLoader = TestLoader()
# Patches for old functions: these functions should be considered obsolete
##############################################################################
-def _makeLoader(prefix, sortUsing, suiteClass=None):
+def _makeLoader(prefix, suiteClass=None):
loader = TestLoader()
- loader.sortTestMethodsUsing = sortUsing
loader.testMethodPrefix = prefix
if suiteClass: loader.suiteClass = suiteClass
return loader
-def getTestCaseNames(testCaseClass, prefix, sortUsing=cmp):
- return _makeLoader(prefix, sortUsing).getTestCaseNames(testCaseClass)
+def getTestCaseNames(testCaseClass, prefix):
+ return _makeLoader(prefix).getTestCaseNames(testCaseClass)
-def makeSuite(testCaseClass, prefix='test', sortUsing=cmp, suiteClass=TestSuite):
- return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromTestCase(testCaseClass)
+def makeSuite(testCaseClass, prefix='test',suiteClass=TestSuite):
+ return _makeLoader(prefix, suiteClass).loadTestsFromTestCase(testCaseClass)
-def findTestCases(module, prefix='test', sortUsing=cmp, suiteClass=TestSuite):
- return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromModule(module)
+def findTestCases(module, prefix='test',suiteClass=TestSuite):
+ return _makeLoader(prefix, suiteClass).loadTestsFromModule(module)
##############################################################################
@@ -825,8 +817,8 @@ class TextTestRunner:
self.stream.writeln()
if not result.wasSuccessful():
self.stream.write("FAILED (")
- failed, errored, skipped = map(len, \
- (result.failures, result.errors, result.skipped))
+ failed, errored, skipped = list(map(len, \
+ (result.failures, result.errors, result.skipped)))
if failed:
self.stream.write("failures=%d" % failed)
if errored:
@@ -871,7 +863,7 @@ Examples:
argv=None, testRunner=None, testLoader=defaultTestLoader):
if type(module) == type(''):
self.module = __import__(module)
- for part in string.split(module,'.')[1:]:
+ for part in module.split('.')[1:]:
self.module = getattr(self.module, part)
else:
self.module = module
@@ -886,8 +878,8 @@ Examples:
self.runTests()
def usageExit(self, msg=None):
- if msg: print msg
- print self.USAGE % self.__dict__
+ if msg: print(msg)
+ print(self.USAGE % self.__dict__)
sys.exit(2)
def parseArgs(self, argv):
@@ -910,7 +902,7 @@ Examples:
else:
self.testNames = (self.defaultTest,)
self.createTests()
- except getopt.error, msg:
+ except getopt.error as msg:
self.usageExit(msg)
def createTests(self):
diff --git a/test/runtests.py b/test/runtests.py
index c48bd1d..78a5974 100644
--- a/test/runtests.py
+++ b/test/runtests.py
@@ -54,7 +54,7 @@ def parse_args():
return (descriptions,verbosity)
def usage():
- print __doc__
+ print(__doc__)
if __name__ == '__main__':
main()
diff --git a/test/test_byterange.py b/test/test_byterange.py
index 0f75807..0863be8 100644
--- a/test/test_byterange.py
+++ b/test/test_byterange.py
@@ -24,8 +24,11 @@
# $Id: test_byterange.py,v 1.6 2004/03/31 17:02:00 mstenner Exp $
import sys
+import six
-from cStringIO import StringIO
+from io import StringIO
+
+import urlgrabber
from urlgrabber.byterange import RangeableFileObject
from base_test_code import *
@@ -37,7 +40,7 @@ class RangeableFileObjectTestCase(TestCase):
# 0 1 2 3 4 5 6 7 8 9
# 0123456789012345678901234567890123456789012345678901234567 890123456789012345678901234567890
self.test = 'Why cannot we write the entire 24 volumes of Encyclopaedia\nBrittanica on the head of a pin?\n'
- self.fo = StringIO(self.test)
+ self.fo = StringIO(unicode(self.test) if not six.PY3 else self.test)
self.rfo = RangeableFileObject(self.fo, (20,69))
def tearDown(self):
@@ -61,7 +64,8 @@ class RangeableFileObjectTestCase(TestCase):
def test_readall(self):
"""RangeableFileObject.read(): to end of file."""
- rfo = RangeableFileObject(StringIO(self.test),(11,))
+ text_compat = unicode(self.test) if not six.PY3 else self.test
+ rfo = RangeableFileObject(StringIO(text_compat),(11,))
self.assertEquals(self.test[11:],rfo.read())
def test_readline(self):
diff --git a/test/test_grabber.py b/test/test_grabber.py
index 8e45d25..bd36d66 100644
--- a/test/test_grabber.py
+++ b/test/test_grabber.py
@@ -24,11 +24,13 @@
# $Id: test_grabber.py,v 1.31 2006/12/08 00:14:16 mstenner Exp $
import sys
+import six
import os
-import string, tempfile, random, cStringIO, os
-import urllib2
+import tempfile, random, os
+from six.moves import urllib
import socket
+from io import StringIO
from base_test_code import *
import urlgrabber
@@ -41,12 +43,12 @@ class FileObjectTests(TestCase):
def setUp(self):
self.filename = tempfile.mktemp()
- fo = file(self.filename, 'wb')
- fo.write(reference_data)
+ fo = open(self.filename, 'wb')
+ fo.write(reference_data.encode('utf-8'))
fo.close()
- self.fo_input = cStringIO.StringIO(reference_data)
- self.fo_output = cStringIO.StringIO()
+ self.fo_input = StringIO(unicode(reference_data) if not six.PY3 else reference_data)
+ self.fo_output = StringIO()
(url, parts) = grabber.default_grabber.opts.urlparser.parse(
self.filename, grabber.default_grabber.opts)
self.wrapper = grabber.PyCurlFileObject(
@@ -73,7 +75,7 @@ class FileObjectTests(TestCase):
def test_readlines(self):
"PyCurlFileObject .readlines() method"
li = self.wrapper.readlines()
- self.fo_output.write(string.join(li, ''))
+ self.fo_output.write(''.join(li))
self.assert_(reference_data == self.fo_output.getvalue())
def test_smallread(self):
@@ -90,7 +92,7 @@ class HTTPTests(TestCase):
filename = tempfile.mktemp()
grabber.urlgrab(ref_http, filename)
- fo = file(filename, 'rb')
+ fo = open(filename, 'rb' if not six.PY3 else 'r')
contents = fo.read()
fo.close()
@@ -136,7 +138,7 @@ class URLGrabberTestCase(TestCase):
def setUp(self):
- self.meter = text_progress_meter( fo=cStringIO.StringIO() )
+ self.meter = text_progress_meter( fo=StringIO() )
pass
def tearDown(self):
@@ -149,7 +151,7 @@ class URLGrabberTestCase(TestCase):
values into the URLGrabber constructor and checks that
they've been set properly.
"""
- opener = urllib2.OpenerDirector()
+ opener = urllib.request.OpenerDirector()
g = URLGrabber( progress_obj=self.meter,
throttle=0.9,
bandwidth=20,
@@ -225,13 +227,13 @@ class URLParserTestCase(TestCase):
self.assertEquals(parts, urllist[2])
else:
if url == urllist[1] and parts == urllist[2]:
- print 'OK: %s' % urllist[0]
+ print('OK: %s' % urllist[0])
else:
- print 'ERROR: %s' % urllist[0]
- print ' ' + urllist[1]
- print ' ' + url
- print ' ' + urllist[2]
- print ' ' + parts
+ print('ERROR: %s' % urllist[0])
+ print(' ' + urllist[1])
+ print(' ' + url)
+ print(' ' + urllist[2])
+ print(' ' + parts)
url_tests_all = (
@@ -380,7 +382,7 @@ class CheckfuncTestCase(TestCase):
if hasattr(obj, 'filename'):
# we used urlgrab
- fo = file(obj.filename)
+ fo = open(obj.filename)
data = fo.read()
fo.close()
else:
@@ -447,12 +449,12 @@ class RegetTestBase:
except: pass
def _make_half_zero_file(self):
- fo = file(self.filename, 'wb')
- fo.write('0'*self.hl)
+ fo = open(self.filename, 'wb' if not six.PY3 else 'w')
+ fo.write('0'*int(self.hl))
fo.close()
def _read_file(self):
- fo = file(self.filename, 'rb')
+ fo = open(self.filename, 'rb' if not six.PY3 else 'r')
data = fo.read()
fo.close()
return data
@@ -470,7 +472,7 @@ class FTPRegetTests(RegetTestBase, TestCase):
# this tests to see if the server is available. If it's not,
# then these tests will be skipped
try:
- fo = urllib2.urlopen(self.url).close()
+ fo = urllib.request.urlopen(self.url).close()
except IOError:
self.skip()
@@ -480,8 +482,8 @@ class FTPRegetTests(RegetTestBase, TestCase):
self.grabber.urlgrab(self.url, self.filename, reget='simple')
data = self._read_file()
- self.assertEquals(data[:self.hl], '0'*self.hl)
- self.assertEquals(data[self.hl:], self.ref[self.hl:])
+ self.assertEquals(data[:int(self.hl)], '0'*int(self.hl))
+ self.assertEquals(data[int(self.hl):], self.ref[int(self.hl):])
class HTTPRegetTests(FTPRegetTests):
def setUp(self):
@@ -498,8 +500,8 @@ class HTTPRegetTests(FTPRegetTests):
self.grabber.urlgrab(self.url, self.filename, reget='check_timestamp')
data = self._read_file()
- self.assertEquals(data[:self.hl], '0'*self.hl)
- self.assertEquals(data[self.hl:], self.ref[self.hl:])
+ self.assertEquals(data[:int(self.hl)], '0'*int(self.hl))
+ self.assertEquals(data[int(self.hl):], self.ref[int(self.hl):])
except NotImplementedError:
self.skip()
@@ -521,7 +523,7 @@ class FileRegetTests(HTTPRegetTests):
def setUp(self):
self.ref = short_reference_data
tmp = tempfile.mktemp()
- tmpfo = file(tmp, 'wb')
+ tmpfo = open(tmp, 'wb' if not six.PY3 else 'w')
tmpfo.write(self.ref)
tmpfo.close()
self.tmp = tmp
@@ -545,7 +547,7 @@ class ProFTPDSucksTests(TestCase):
def setUp(self):
self.url = ref_proftp
try:
- fo = urllib2.urlopen(self.url).close()
+ fo = urllib.request.urlopen(self.url).close()
except IOError:
self.skip()
@@ -592,7 +594,7 @@ class ProxyFTPAuthTests(ProxyHTTPAuthTests):
if not self.have_proxy():
self.skip()
try:
- fo = urllib2.urlopen(self.url).close()
+ fo = urllib.request.urlopen(self.url).close()
except IOError:
self.skip()
self.g = URLGrabber()
diff --git a/test/test_mirror.py b/test/test_mirror.py
index 7f493d0..c46cd33 100644
--- a/test/test_mirror.py
+++ b/test/test_mirror.py
@@ -24,8 +24,9 @@
# $Id: test_mirror.py,v 1.12 2005/10/22 21:57:27 mstenner Exp $
import sys
+import six
import os
-import string, tempfile, random, cStringIO, os
+import string, tempfile, random, os
import urlgrabber.grabber
from urlgrabber.grabber import URLGrabber, URLGrabError, URLGrabberOptions
@@ -268,7 +269,8 @@ class ActionTests(TestCase):
self.assertEquals(self.g.calls, expected_calls)
self.assertEquals(urlgrabber.mirror.DEBUG.logs, expected_logs)
-import thread, socket
+from six.moves import _thread as thread
+import socket
LOCALPORT = 'localhost', 2000
class HttpReplyCode(TestCase):
@@ -282,11 +284,14 @@ class HttpReplyCode(TestCase):
while 1:
c, a = s.accept()
if self.exit: c.close(); break
- while not c.recv(4096).endswith('\r\n\r\n'): pass
- c.sendall('HTTP/1.1 %d %s\r\n' % self.reply)
+ ending_compat = '\r\n\r\n' if not six.PY3 else b'\r\n\r\n'
+ while not c.recv(4096).endswith(ending_compat): pass
+ http_compat = 'HTTP/1.1 %d %s\r\n' % self.reply
+ c.sendall(http_compat if not six.PY3 else http_compat.encode('utf-8'))
if self.content is not None:
- c.sendall('Content-Length: %d\r\n\r\n' % len(self.content))
- c.sendall(self.content)
+ cont_length_compat = 'Content-Length: %d\r\n\r\n' % len(self.content)
+ c.sendall(cont_length_compat if not six.PY3 else cont_length_compat.encode('utf-8'))
+ c.sendall(self.content if not six.PY3 else self.content.encode('utf-8'))
c.close()
s.close()
self.exit = False

1175
port-to-python3.patch
View File

File diff suppressed because it is too large Load Diff

15
python-urlgrabber.spec
Unescape View File

@ -3,22 +3,19 @@
Summary: A high-level cross-protocol url-grabber Summary: A high-level cross-protocol url-grabber
Name: python-urlgrabber Name: python-urlgrabber
Version: 3.10.1 Version: 3.10.1
Release: 5%{?dist} Release: 6%{?dist}
Source0: http://urlgrabber.baseurl.org/download/urlgrabber-%{version}.tar.gz Source0: http://urlgrabber.baseurl.org/download/urlgrabber-%{version}.tar.gz
Patch1: urlgrabber-HEAD.patch Patch1: urlgrabber-HEAD.patch
Patch2: BZ-1051554-speed-on-404-mirror.patch Patch2: BZ-1051554-speed-on-404-mirror.patch
Patch3: port-to-python3.patch
Patch4: port-tests-to-python3.patch
Patch5: urlgrabber-stringio.patch
License: LGPLv2+ License: LGPLv2+
Group: Development/Libraries Group: Development/Libraries
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
BuildArch: noarch BuildArch: noarch
BuildRequires: python-devel, python-pycurl, python-six BuildRequires: python-devel, python-pycurl
Url: http://urlgrabber.baseurl.org/ Url: http://urlgrabber.baseurl.org/
Provides: urlgrabber = %{version}-%{release} Provides: urlgrabber = %{version}-%{release}
Requires: python-pycurl, python-six Requires: python-pycurl
%description %description
A high-level cross-protocol url-grabber for python supporting HTTP, FTP A high-level cross-protocol url-grabber for python supporting HTTP, FTP
@ -29,9 +26,6 @@ authentication, proxies and more.
%setup -q -n urlgrabber-%{version} %setup -q -n urlgrabber-%{version}
%patch1 -p1 %patch1 -p1
%patch2 -p1 %patch2 -p1
%patch3 -p1
%patch4 -p1
%patch5 -p1
%build %build
python setup.py build python setup.py build
@ -54,6 +48,9 @@ rm -rf $RPM_BUILD_ROOT
%attr(0755,root,root) %{_libexecdir}/urlgrabber-ext-down %attr(0755,root,root) %{_libexecdir}/urlgrabber-ext-down
%changelog %changelog
* Fri Oct 03 2014 Valentina Mukhamedzhanova <vmukhame@redhat.com> - 3.10.1-6
- Revert porting to Python 3.
* Tue Sep 09 2014 Tomas Radej <tradej@redhat.com> - 3.10.1-5 * Tue Sep 09 2014 Tomas Radej <tradej@redhat.com> - 3.10.1-5
- Really fixed UTF behaviour - Really fixed UTF behaviour

433
urlgrabber-stringio.patch
Unescape View File

@ -1,433 +0,0 @@
diff --git a/test/test_grabber.py b/test/test_grabber.py
index bd36d66..bd54329 100644
--- a/test/test_grabber.py
+++ b/test/test_grabber.py
@@ -42,7 +42,7 @@ from urlgrabber.progress import text_progress_meter
class FileObjectTests(TestCase):
def setUp(self):
- self.filename = tempfile.mktemp()
+ _, self.filename = tempfile.mkstemp()
fo = open(self.filename, 'wb')
fo.write(reference_data.encode('utf-8'))
fo.close()
@@ -61,35 +61,36 @@ class FileObjectTests(TestCase):
def test_readall(self):
"PYCurlFileObject .read() method"
s = self.wrapper.read()
- self.fo_output.write(s)
+ self.fo_output.write(unicode(s) if not six.PY3 else s)
self.assert_(reference_data == self.fo_output.getvalue())
def test_readline(self):
"PyCurlFileObject .readline() method"
while 1:
s = self.wrapper.readline()
- self.fo_output.write(s)
+ self.fo_output.write(unicode(s) if not six.PY3 else s)
if not s: break
self.assert_(reference_data == self.fo_output.getvalue())
def test_readlines(self):
"PyCurlFileObject .readlines() method"
li = self.wrapper.readlines()
- self.fo_output.write(''.join(li))
+ out = ''.join(li)
+ self.fo_output.write(unicode(out) if not six.PY3 else out)
self.assert_(reference_data == self.fo_output.getvalue())
def test_smallread(self):
"PyCurlFileObject .read(N) with small N"
while 1:
s = self.wrapper.read(23)
- self.fo_output.write(s)
+ self.fo_output.write(unicode(s) if not six.PY3 else s)
if not s: break
self.assert_(reference_data == self.fo_output.getvalue())
class HTTPTests(TestCase):
def test_reference_file(self):
"download reference file via HTTP"
- filename = tempfile.mktemp()
+ _, filename = tempfile.mkstemp()
grabber.urlgrab(ref_http, filename)
fo = open(filename, 'rb' if not six.PY3 else 'r')
@@ -123,7 +124,7 @@ class URLGrabberModuleTestCase(TestCase):
def test_urlgrab(self):
"module-level urlgrab() function"
- outfile = tempfile.mktemp()
+ _, outfile = tempfile.mkstemp()
filename = urlgrabber.urlgrab('http://www.python.org',
filename=outfile)
os.unlink(outfile)
@@ -367,7 +368,7 @@ class CheckfuncTestCase(TestCase):
def setUp(self):
cf = (self._checkfunc, ('foo',), {'bar': 'baz'})
self.g = grabber.URLGrabber(checkfunc=cf)
- self.filename = tempfile.mktemp()
+ _, self.filename = tempfile.mkstemp()
self.data = short_reference_data
def tearDown(self):
@@ -440,7 +441,7 @@ class RegetTestBase:
def setUp(self):
self.ref = short_reference_data
self.grabber = grabber.URLGrabber(reget='check_timestamp')
- self.filename = tempfile.mktemp()
+ _, self.filename = tempfile.mkstemp()
self.hl = len(self.ref) / 2
self.url = 'OVERRIDE THIS'
@@ -522,7 +523,7 @@ class HTTPRegetTests(FTPRegetTests):
class FileRegetTests(HTTPRegetTests):
def setUp(self):
self.ref = short_reference_data
- tmp = tempfile.mktemp()
+ _, tmp = tempfile.mkstemp()
tmpfo = open(tmp, 'wb' if not six.PY3 else 'w')
tmpfo.write(self.ref)
tmpfo.close()
@@ -534,7 +535,7 @@ class FileRegetTests(HTTPRegetTests):
self.grabber = grabber.URLGrabber(reget='check_timestamp',
copy_local=1)
- self.filename = tempfile.mktemp()
+ _, self.filename = tempfile.mkstemp()
self.hl = len(self.ref) / 2
def tearDown(self):
diff --git a/test/test_mirror.py b/test/test_mirror.py
index c46cd33..b923dd1 100644
--- a/test/test_mirror.py
+++ b/test/test_mirror.py
@@ -50,7 +50,7 @@ class BasicTests(TestCase):
def test_urlgrab(self):
"""MirrorGroup.urlgrab"""
- filename = tempfile.mktemp()
+ _, filename = tempfile.mkstemp()
url = 'short_reference'
self.mg.urlgrab(url, filename)
@@ -84,7 +84,7 @@ class SubclassTests(TestCase):
def fetchwith(self, mgclass):
self.mg = mgclass(self.g, self.fullmirrors)
- filename = tempfile.mktemp()
+ _, filename = tempfile.mkstemp()
url = 'short_reference'
self.mg.urlgrab(url, filename)
@@ -137,7 +137,7 @@ class BadMirrorTests(TestCase):
def test_simple_grab(self):
"""test that a bad mirror raises URLGrabError"""
- filename = tempfile.mktemp()
+ _, filename = tempfile.mkstemp()
url = 'reference'
self.assertRaises(URLGrabError, self.mg.urlgrab, url, filename)
@@ -150,7 +150,7 @@ class FailoverTests(TestCase):
def test_simple_grab(self):
"""test that a the MG fails over past a bad mirror"""
- filename = tempfile.mktemp()
+ _, filename = tempfile.mkstemp()
url = 'reference'
elist = []
def cb(e, elist=elist): elist.append(e)
diff --git a/urlgrabber/byterange.py b/urlgrabber/byterange.py
index ffaed8e..95287fc 100644
--- a/urlgrabber/byterange.py
+++ b/urlgrabber/byterange.py
@@ -27,7 +27,7 @@ from six.moves import urllib
DEBUG = None
-from io import StringIO
+from io import BytesIO
class RangeError(IOError):
"""Error raised when an unsatisfiable range is requested."""
@@ -238,8 +238,8 @@ class FileRangeHandler(urllib.request.FileHandler):
raise RangeError(9, 'Requested Range Not Satisfiable')
size = (lb - fb)
fo = RangeableFileObject(fo, (fb,lb))
- headers = email.message.Message(StringIO(
- 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
+ headers = email.message.Message(BytesIO(
+ b'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
(mtype or 'text/plain', size, modified)))
return urllib.addinfourl(fo, headers, 'file:'+file)
@@ -323,13 +323,13 @@ class FTPRangeHandler(urllib.request.FTPHandler):
fp = RangeableFileObject(fp, (0,retrlen))
# -- range support modifications end here
- headers = ""
+ headers = b""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
- headers += "Content-Type: %s\n" % mtype
+ headers += b"Content-Type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
- headers += "Content-Length: %d\n" % retrlen
- sf = StringIO(headers)
+ headers += b"Content-Length: %d\n" % retrlen
+ sf = BytesIO(headers)
headers = email.message.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors as msg:
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 35c091e..69cd113 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -516,7 +516,7 @@ from six.moves import urllib
from six.moves.http_client import responses, HTTPException
from urlgrabber.byterange import range_tuple_normalize, range_tuple_to_header, RangeError
-from io import StringIO
+from io import BytesIO
try:
import xattr
@@ -1235,7 +1235,7 @@ default_grabber = URLGrabber()
class PyCurlFileObject(object):
def __init__(self, url, filename, opts):
self.fo = None
- self._hdr_dump = ''
+ self._hdr_dump = b''
self._parsed_hdr = None
self.url = url
self.scheme = urllib.parse.urlsplit(self.url)[0]
@@ -1246,7 +1246,7 @@ class PyCurlFileObject(object):
if self.opts.reget == 'check_timestamp':
raise NotImplementedError("check_timestamp regets are not implemented in this ver of urlgrabber. Please report this.")
self._complete = False
- self._rbuf = ''
+ self._rbuf = b''
self._rbufsize = 1024*8
self._ttime = time.time()
self._tsize = 0
@@ -1298,15 +1298,9 @@ class PyCurlFileObject(object):
start = self._range[0] - pos
stop = self._range[1] - pos
if start < len(buf) and stop > 0:
- if not six.PY3 or isinstance(self.fo, StringIO):
- self.fo.write(buf[max(start, 0):stop].decode('utf-8'))
- else:
- self.fo.write(buf[max(start, 0):stop])
+ self.fo.write(buf[max(start, 0):stop])
else:
- if not six.PY3 or isinstance(self.fo, StringIO):
- self.fo.write(buf.decode('utf-8'))
- else:
- self.fo.write(buf)
+ self.fo.write(buf)
except IOError as e:
self._cb_error = URLGrabError(16, exception2msg(e))
return -1
@@ -1316,7 +1310,7 @@ class PyCurlFileObject(object):
def _hdr_retrieve(self, buf):
if self._hdr_ended:
- self._hdr_dump = ''
+ self._hdr_dump = b''
self.size = 0
self._hdr_ended = False
@@ -1328,12 +1322,12 @@ class PyCurlFileObject(object):
# but we can't do that w/o making it do 2 connects, which sucks
# so we cheat and stuff it in here in the hdr_retrieve
if self.scheme in ['http','https']:
- content_length_str = 'content-length:' if not six.PY3 else b'content-length:'
+ content_length_str = b'content-length:'
if buf.lower().find(content_length_str) != -1:
- split_str = ':' if not six.PY3 else b':'
+ split_str = b':'
length = buf.split(split_str)[1]
self.size = int(length)
- elif (self.append or self.opts.range) and self._hdr_dump == '' and b' 200 ' in buf:
+ elif (self.append or self.opts.range) and self._hdr_dump == b'' and b' 200 ' in buf:
# reget was attempted but server sends it all
# undo what we did in _build_range()
self.append = False
@@ -1349,20 +1343,19 @@ class PyCurlFileObject(object):
if len(s) >= 14:
s = None # ignore MDTM responses
elif buf.startswith(b'150 '):
- s = parse150(buf if not six.PY3 else buf.decode('utf-8'))
+ s = parse150(buf.decode('utf-8')) # Necessary in Python 3, doesn't hurt in Python 2
if s:
self.size = int(s)
- location_str = 'location' if not six.PY3 else b'location'
+ location_str = b'location'
if buf.lower().find(location_str) != -1:
- buf_compat = buf if not six.PY3 else buf.decode('utf-8')
- location = ':'.join(buf_compat.split(':')[1:])
+ location = b':'.join(buf.split(b':')[1:])
location = location.strip()
self.scheme = urllib.parse.urlsplit(location)[0]
self.url = location
- self._hdr_dump += buf if not six.PY3 else buf.decode('utf-8')
- end_str = '\r\n' if not six.PY3 else b'\r\n'
+ self._hdr_dump += buf
+ end_str = b'\r\n'
if len(self._hdr_dump) != 0 and buf == end_str:
self._hdr_ended = True
if DEBUG: DEBUG.debug('header ended:')
@@ -1374,12 +1367,12 @@ class PyCurlFileObject(object):
def _return_hdr_obj(self):
if self._parsed_hdr:
return self._parsed_hdr
- statusend = self._hdr_dump.find('\n')
+ statusend = self._hdr_dump.find(b'\n')
statusend += 1 # ridiculous as it may seem.
- hdrfp = StringIO()
+ hdrfp = BytesIO()
hdrfp.write(self._hdr_dump[statusend:])
hdrfp.seek(0)
- self._parsed_hdr = Message(hdrfp)
+ self._parsed_hdr = Message(hdrfp)
return self._parsed_hdr
hdr = property(_return_hdr_obj)
@@ -1709,7 +1702,7 @@ class PyCurlFileObject(object):
return (fo, hdr)
def _do_grab(self):
- """dump the file to a filename or StringIO buffer"""
+ """dump the file to a filename or BytesIO buffer"""
if self._complete:
return
@@ -1739,7 +1732,7 @@ class PyCurlFileObject(object):
self._prog_basename = 'MEMORY'
- self.fo = StringIO()
+ self.fo = BytesIO()
# if this is to be a tempfile instead....
# it just makes crap in the tempdir
#fh, self._temp_name = mkstemp()
@@ -1778,7 +1771,7 @@ class PyCurlFileObject(object):
raise err
# re open it
try:
- self.fo = open(self.filename, 'r')
+ self.fo = open(self.filename, 'rb')
except IOError as e:
err = URLGrabError(16, _(\
'error opening file from %s, IOError: %s') % (self.url, e))
@@ -1853,7 +1846,7 @@ class PyCurlFileObject(object):
#if self.opts.progress_obj:
# self.opts.progress_obj.update(self._amount_read)
- self._rbuf = ''.join(buf)
+ self._rbuf = b''.join(buf)
return
def _progress_update(self, download_total, downloaded, upload_total, uploaded):
@@ -1888,28 +1881,40 @@ class PyCurlFileObject(object):
def read(self, amt=None):
self._fill_buffer(amt)
if amt is None:
- s, self._rbuf = self._rbuf, ''
+ s, self._rbuf = self._rbuf, b''
else:
s, self._rbuf = self._rbuf[:amt], self._rbuf[amt:]
- return s
+ return s if not six.PY3 else s.decode('utf-8')
def readline(self, limit=-1):
if not self._complete: self._do_grab()
- return self.fo.readline()
+ return self.fo.readline() if not six.PY3 else self.fo.readline().decode('utf-8')
- i = self._rbuf.find('\n')
+ i = self._rbuf.find(b'\n')
while i < 0 and not (0 < limit <= len(self._rbuf)):
L = len(self._rbuf)
self._fill_buffer(L + self._rbufsize)
if not len(self._rbuf) > L: break
- i = self._rbuf.find('\n', L)
+ i = self._rbuf.find(b'\n', L)
if i < 0: i = len(self._rbuf)
else: i = i+1
if 0 <= limit < len(self._rbuf): i = limit
s, self._rbuf = self._rbuf[:i], self._rbuf[i:]
- return s
+ return s if not six.PY3 else s.decode('utf-8')
+
+ # This was added here because we need to wrap self.fo readlines (which will
+ # always return bytes) in correct decoding
+ def readlines(self, *args, **kwargs):
+ if not six.PY3:
+ return [line for line in self.fo.readlines(*args, **kwargs)]
+ else:
+ return self._py3readlines(*args, **kwargs)
+
+ def _py3readlines(self, *args, **kwargs):
+ for line in self.fo.readlines(*args, **kwargs):
+ yield line.decode('utf-8')
def close(self):
if self._prog_running:
@@ -2055,11 +2060,9 @@ def _readlines(fd):
buf = os.read(fd, 4096)
if not buf: return None
# whole lines only, no buffering
- buf_compat = buf if not six.PY3 else buf.decode('utf-8')
- while buf_compat[-1] != '\n':
+ while buf.decode('utf-8')[-1] != '\n':
buf += os.read(fd, 4096)
- buf_compat = buf if not six.PY3 else buf.decode('utf-8')
- return buf_compat[:-1].split('\n')
+ return buf.decode('utf-8')[:-1].split('\n')
import subprocess
@@ -2403,7 +2406,7 @@ class _TH:
if filename and _TH.dirty is None:
try:
now = int(time.time())
- for line in open(filename):
+ for line in open(filename, 'rb'):
host, speed, fail, ts = line.rsplit(' ', 3)
_TH.hosts[host] = int(speed), int(fail), min(int(ts), now)
except IOError: pass
@@ -2415,7 +2418,7 @@ class _TH:
if filename and _TH.dirty is True:
tmp = '%s.%d' % (filename, os.getpid())
try:
- f = open(tmp, 'w')
+ f = open(tmp, 'wb')
for host in _TH.hosts:
f.write(host + ' %d %d %d\n' % _TH.hosts[host])
f.close()
@@ -2536,7 +2539,7 @@ def _file_object_test(filename=None):
if filename is None:
filename = __file__
print('using file "%s" for comparisons' % filename)
- fo = open(filename)
+ fo = open(filename, 'rb')
s_input = fo.read()
fo.close()
@@ -2544,8 +2547,8 @@ def _file_object_test(filename=None):
_test_file_object_readall,
_test_file_object_readline,
_test_file_object_readlines]:
- fo_input = StringIO(s_input)
- fo_output = StringIO()
+ fo_input = BytesIO(s_input)
+ fo_output = BytesIO()
wrapper = PyCurlFileObject(fo_input, None, 0)
print('testing %-30s ' % testfunc.__name__, testfunc(wrapper, fo_output))
s_output = fo_output.getvalue()
Write Preview
Loading…
Cancel
Save