<sigh> actually check in the patch :(

epel9
Seth Vidal 16 years ago
parent 7eafa78073
commit 891a468499

@ -3,7 +3,7 @@
Summary: A high-level cross-protocol url-grabber Summary: A high-level cross-protocol url-grabber
Name: python-urlgrabber Name: python-urlgrabber
Version: 3.9.0 Version: 3.9.0
Release: 5%{?dist} Release: 6%{?dist}
Source0: urlgrabber-%{version}.tar.gz Source0: urlgrabber-%{version}.tar.gz
Patch0: urlgrabber-HEAD.patch Patch0: urlgrabber-HEAD.patch

@ -0,0 +1,110 @@
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index cf51dff..3758799 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -402,6 +402,7 @@ import urllib
import urllib2
import mimetools
import thread
+import types
from stat import * # S_* and ST_*
import pycurl
from ftplib import parse150
@@ -1219,7 +1220,7 @@ class URLGrabberFileObject:
self.append = 0
reget_length = 0
rt = None
- if have_range and self.opts.reget and type(self.filename) == type(''):
+ if have_range and self.opts.reget and type(self.filename) in types.StringTypes:
# we have reget turned on and we're dumping to a file
try:
s = os.stat(self.filename)
@@ -1450,6 +1451,7 @@ class PyCurlFileObject():
self.scheme = urlparse.urlsplit(self.url)[0]
self.filename = filename
self.append = False
+ self.reget_time = None
self.opts = opts
self._complete = False
self.reget_time = None
@@ -1528,11 +1530,12 @@ class PyCurlFileObject():
# defaults we're always going to set
- self.curl_obj.setopt(pycurl.NOPROGRESS, 0)
+ self.curl_obj.setopt(pycurl.NOPROGRESS, False)
self.curl_obj.setopt(pycurl.WRITEFUNCTION, self._retrieve)
self.curl_obj.setopt(pycurl.HEADERFUNCTION, self._hdr_retrieve)
self.curl_obj.setopt(pycurl.PROGRESSFUNCTION, self._progress_update)
- self.curl_obj.setopt(pycurl.FAILONERROR, 1)
+ self.curl_obj.setopt(pycurl.FAILONERROR, True)
+ self.curl_obj.setopt(pycurl.OPT_FILETIME, True)
if DEBUG:
self.curl_obj.setopt(pycurl.VERBOSE, True)
@@ -1540,15 +1543,15 @@ class PyCurlFileObject():
self.curl_obj.setopt(pycurl.USERAGENT, opts.user_agent)
# maybe to be options later
- self.curl_obj.setopt(pycurl.FOLLOWLOCATION, 1)
+ self.curl_obj.setopt(pycurl.FOLLOWLOCATION, True)
self.curl_obj.setopt(pycurl.MAXREDIRS, 5)
- self.curl_obj.setopt(pycurl.CONNECTTIMEOUT, 30)
# timeouts
timeout = 300
if opts.timeout:
timeout = int(opts.timeout)
- self.curl_obj.setopt(pycurl.TIMEOUT, timeout)
+ self.curl_obj.setopt(pycurl.CONNECTTIMEOUT, timeout)
+
# ssl options
if self.scheme == 'https':
if opts.ssl_ca_cert: # this may do ZERO with nss according to curl docs
@@ -1607,18 +1610,21 @@ class PyCurlFileObject():
# to other URLGrabErrors from
# http://curl.haxx.se/libcurl/c/libcurl-errors.html
# this covers e.args[0] == 22 pretty well - which will be common
+ if e.args[0] == 28:
+ err = URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
+ err.url = self.url
+ raise err
+ code = self.http_code
if str(e.args[1]) == '': # fake it until you make it
msg = 'HTTP Error %s : %s ' % (self.http_code, self.url)
else:
msg = str(e.args[1])
err = URLGrabError(14, msg)
- err.code = self.http_code
+ err.code = code
err.exception = e
raise err
def _do_open(self):
- self.append = False
- self.reget_time = None
self.curl_obj = _curl_cache
self.curl_obj.reset() # reset all old settings away, just in case
# setup any ranges
@@ -1630,11 +1636,9 @@ class PyCurlFileObject():
pass
def _build_range(self):
- self.reget_time = None
- self.append = False
reget_length = 0
rt = None
- if self.opts.reget and type(self.filename) == type(''):
+ if self.opts.reget and type(self.filename) in types.StringTypes:
# we have reget turned on and we're dumping to a file
try:
s = os.stat(self.filename)
@@ -1729,7 +1733,7 @@ class PyCurlFileObject():
if self.filename:
self._prog_reportname = str(self.filename)
self._prog_basename = os.path.basename(self.filename)
-
+
if self.append: mode = 'ab'
else: mode = 'wb'
Loading…
Cancel
Save