rpms
/
python-urlgrabber
20
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update to latest HEAD.

Browse Source
epel9
Zdeněk Pavlas 13 years ago
parent 06d3b13e99
commit c0934722e0
2 changed files with 292 additions and 79 deletions
Show Stats Download Patch File Download Diff File

6
python-urlgrabber.spec
Unescape View File

@ -3,7 +3,7 @@
Summary: A high-level cross-protocol url-grabber
Name: python-urlgrabber
Version: 3.9.1
Release: 19%{?dist}
Release: 20%{?dist}
Source0: urlgrabber-%{version}.tar.gz
Patch1: urlgrabber-HEAD.patch
@ -44,6 +44,10 @@ rm -rf $RPM_BUILD_ROOT
%attr(0755,root,root) %{_libexecdir}/urlgrabber-ext-down
%changelog
* Tue Sep 4 2012 Zdeněk Pavlas <zpavlas@redhat.com> - 3.9.1-20
- Update to latest HEAD.
- Fixed BZ 851178, 854075.
* Mon Aug 27 2012 Zdeněk Pavlas <zpavlas@redhat.com> - 3.9.1-19
- timedhosts: defer 1st update until a 1MB+ download. BZ 851178

365
urlgrabber-HEAD.patch
Unescape View File

@ -73,10 +73,10 @@ index 518e512..09cd896 100644
print __doc__
diff --git a/scripts/urlgrabber-ext-down b/scripts/urlgrabber-ext-down
new file mode 100755
index 0000000..3da55a4
index 0000000..3dafb12
--- /dev/null
+++ b/scripts/urlgrabber-ext-down
@@ -0,0 +1,72 @@
@@ -0,0 +1,75 @@
+#! /usr/bin/python
+# A very simple external downloader
+# Copyright 2011-2012 Zdenek Pavlas
@ -134,18 +134,21 @@ index 0000000..3da55a4
+ if opts.progress_obj:
+ opts.progress_obj = ProxyProgress()
+ opts.progress_obj._id = cnt
+ tm = time.time()
+
+ dlsz = dltm = 0
+ try:
+ fo = PyCurlFileObject(opts.url, opts.filename, opts)
+ fo._do_grab()
+ fo.fo.close()
+ size = fo._amount_read
+ dlsz = size - fo._reget_length
+ if fo._tm_last:
+ dlsz = fo._tm_last[0] - fo._tm_first[0]
+ dltm = fo._tm_last[1] - fo._tm_first[1]
+ ug_err = 'OK'
+ except URLGrabError, e:
+ size = dlsz = 0
+ size = 0
+ ug_err = '%d %s' % e.args
+ write('%d %d %d %.3f %s\n', opts._id, size, dlsz, time.time() - tm, ug_err)
+ write('%d %d %d %.3f %s\n', opts._id, size, dlsz, dltm, ug_err)
+
+if __name__ == '__main__':
+ main()
@ -233,7 +236,7 @@ index 3e5f3b7..8eeaeda 100644
return (fb,lb)
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index e090e90..daa478d 100644
index e090e90..01218b0 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -49,11 +49,26 @@ GENERAL ARGUMENTS (kwargs)
@ -678,7 +681,7 @@ index e090e90..daa478d 100644
if scheme == 'file' and not opts.copy_local:
# just return the name of the local file - don't make a
# copy currently
@@ -950,41 +1114,49 @@ class URLGrabber:
@@ -950,41 +1114,51 @@ class URLGrabber:
elif not opts.range:
if not opts.checkfunc is None:
@ -700,11 +703,13 @@ index e090e90..daa478d 100644
+ return filename
+
def retryfunc(opts, url, filename):
+ tm = time.time()
fo = PyCurlFileObject(url, filename, opts)
try:
fo._do_grab()
+ _TH.update(url, fo._amount_read - fo._reget_length, time.time() - tm, None)
+ if fo._tm_last:
+ dlsz = fo._tm_last[0] - fo._tm_first[0]
+ dltm = fo._tm_last[1] - fo._tm_first[1]
+ _TH.update(url, dlsz, dltm, None)
if not opts.checkfunc is None:
- cb_func, cb_args, cb_kwargs = \
- self._make_callback(opts.checkfunc)
@ -743,7 +748,7 @@ index e090e90..daa478d 100644
if limit is not None:
limit = limit + 1
@@ -1000,12 +1172,8 @@ class URLGrabber:
@@ -1000,12 +1174,8 @@ class URLGrabber:
else: s = fo.read(limit)
if not opts.checkfunc is None:
@ -758,7 +763,7 @@ index e090e90..daa478d 100644
finally:
fo.close()
return s
@@ -1020,6 +1188,7 @@ class URLGrabber:
@@ -1020,6 +1190,7 @@ class URLGrabber:
return s
def _make_callback(self, callback_obj):
@ -766,7 +771,7 @@ index e090e90..daa478d 100644
if callable(callback_obj):
return callback_obj, (), {}
else:
@@ -1030,7 +1199,7 @@ class URLGrabber:
@@ -1030,7 +1201,7 @@ class URLGrabber:
default_grabber = URLGrabber()
@ -775,13 +780,15 @@ index e090e90..daa478d 100644
def __init__(self, url, filename, opts):
self.fo = None
self._hdr_dump = ''
@@ -1052,10 +1221,11 @@ class PyCurlFileObject():
@@ -1052,10 +1223,13 @@ class PyCurlFileObject():
self._reget_length = 0
self._prog_running = False
self._error = (None, None)
- self.size = None
+ self.size = 0
+ self._hdr_ended = False
+ self._tm_first = None
+ self._tm_last = None
self._do_open()
-
@ -789,7 +796,20 @@ index e090e90..daa478d 100644
def __getattr__(self, name):
"""This effectively allows us to wrap at the instance level.
Any attribute not found in _this_ object will be searched for
@@ -1085,9 +1255,14 @@ class PyCurlFileObject():
@@ -1067,6 +1241,12 @@ class PyCurlFileObject():
def _retrieve(self, buf):
try:
+ tm = self._amount_read + len(buf), time.time()
+ if self._tm_first is None:
+ self._tm_first = tm
+ else:
+ self._tm_last = tm
+
if not self._prog_running:
if self.opts.progress_obj:
size = self.size + self._reget_length
@@ -1085,9 +1265,14 @@ class PyCurlFileObject():
return -1
def _hdr_retrieve(self, buf):
@ -805,7 +825,7 @@ index e090e90..daa478d 100644
try:
self._hdr_dump += buf
# we have to get the size before we do the progress obj start
@@ -1104,7 +1279,17 @@ class PyCurlFileObject():
@@ -1104,7 +1289,17 @@ class PyCurlFileObject():
s = parse150(buf)
if s:
self.size = int(s)
@ -824,7 +844,7 @@ index e090e90..daa478d 100644
return len(buf)
except KeyboardInterrupt:
return pycurl.READFUNC_ABORT
@@ -1113,8 +1298,10 @@ class PyCurlFileObject():
@@ -1113,8 +1308,10 @@ class PyCurlFileObject():
if self._parsed_hdr:
return self._parsed_hdr
statusend = self._hdr_dump.find('\n')
@ -835,7 +855,7 @@ index e090e90..daa478d 100644
self._parsed_hdr = mimetools.Message(hdrfp)
return self._parsed_hdr
@@ -1127,6 +1314,9 @@ class PyCurlFileObject():
@@ -1127,6 +1324,9 @@ class PyCurlFileObject():
if not opts:
opts = self.opts
@ -845,7 +865,7 @@ index e090e90..daa478d 100644
# defaults we're always going to set
self.curl_obj.setopt(pycurl.NOPROGRESS, False)
@@ -1136,11 +1326,21 @@ class PyCurlFileObject():
@@ -1136,11 +1336,21 @@ class PyCurlFileObject():
self.curl_obj.setopt(pycurl.PROGRESSFUNCTION, self._progress_update)
self.curl_obj.setopt(pycurl.FAILONERROR, True)
self.curl_obj.setopt(pycurl.OPT_FILETIME, True)
@ -867,7 +887,7 @@ index e090e90..daa478d 100644
# maybe to be options later
self.curl_obj.setopt(pycurl.FOLLOWLOCATION, True)
@@ -1148,9 +1348,11 @@ class PyCurlFileObject():
@@ -1148,9 +1358,11 @@ class PyCurlFileObject():
# timeouts
timeout = 300
@ -882,7 +902,7 @@ index e090e90..daa478d 100644
# ssl options
if self.scheme == 'https':
@@ -1158,13 +1360,16 @@ class PyCurlFileObject():
@@ -1158,13 +1370,16 @@ class PyCurlFileObject():
self.curl_obj.setopt(pycurl.CAPATH, opts.ssl_ca_cert)
self.curl_obj.setopt(pycurl.CAINFO, opts.ssl_ca_cert)
self.curl_obj.setopt(pycurl.SSL_VERIFYPEER, opts.ssl_verify_peer)
@ -900,7 +920,7 @@ index e090e90..daa478d 100644
if opts.ssl_cert_type:
self.curl_obj.setopt(pycurl.SSLCERTTYPE, opts.ssl_cert_type)
if opts.ssl_key_pass:
@@ -1187,28 +1392,26 @@ class PyCurlFileObject():
@@ -1187,28 +1402,26 @@ class PyCurlFileObject():
if hasattr(opts, 'raw_throttle') and opts.raw_throttle():
self.curl_obj.setopt(pycurl.MAX_RECV_SPEED_LARGE, int(opts.raw_throttle()))
@ -945,7 +965,7 @@ index e090e90..daa478d 100644
# our url
self.curl_obj.setopt(pycurl.URL, self.url)
@@ -1228,12 +1431,14 @@ class PyCurlFileObject():
@@ -1228,12 +1441,14 @@ class PyCurlFileObject():
code = self.http_code
errcode = e.args[0]
@ -962,7 +982,7 @@ index e090e90..daa478d 100644
# this is probably wrong but ultimately this is what happens
# we have a legit http code and a pycurl 'writer failed' code
@@ -1244,23 +1449,23 @@ class PyCurlFileObject():
@@ -1244,23 +1459,23 @@ class PyCurlFileObject():
raise KeyboardInterrupt
elif errcode == 28:
@ -993,7 +1013,7 @@ index e090e90..daa478d 100644
# this is probably wrong but ultimately this is what happens
# we have a legit http code and a pycurl 'writer failed' code
# which almost always means something aborted it from outside
@@ -1272,33 +1477,94 @@ class PyCurlFileObject():
@@ -1272,33 +1487,94 @@ class PyCurlFileObject():
elif errcode == 58:
msg = _("problem with the local client certificate")
err = URLGrabError(14, msg)
@ -1095,7 +1115,7 @@ index e090e90..daa478d 100644
def _do_open(self):
self.curl_obj = _curl_cache
@@ -1333,7 +1599,11 @@ class PyCurlFileObject():
@@ -1333,7 +1609,11 @@ class PyCurlFileObject():
if self.opts.range:
rt = self.opts.range
@ -1108,7 +1128,7 @@ index e090e90..daa478d 100644
if rt:
header = range_tuple_to_header(rt)
@@ -1434,21 +1704,46 @@ class PyCurlFileObject():
@@ -1434,21 +1714,46 @@ class PyCurlFileObject():
#fh, self._temp_name = mkstemp()
#self.fo = open(self._temp_name, 'wb')
@ -1162,7 +1182,7 @@ index e090e90..daa478d 100644
else:
#self.fo = open(self._temp_name, 'r')
self.fo.seek(0)
@@ -1526,17 +1821,20 @@ class PyCurlFileObject():
@@ -1526,17 +1831,20 @@ class PyCurlFileObject():
if self._prog_running:
downloaded += self._reget_length
self.opts.progress_obj.update(downloaded)
@ -1188,7 +1208,7 @@ index e090e90..daa478d 100644
msg = _("Downloaded more than max size for %s: %s > %s") \
% (self.url, cur, max_size)
@@ -1544,13 +1842,6 @@ class PyCurlFileObject():
@@ -1544,13 +1852,6 @@ class PyCurlFileObject():
return True
return False
@ -1202,7 +1222,7 @@ index e090e90..daa478d 100644
def read(self, amt=None):
self._fill_buffer(amt)
if amt is None:
@@ -1582,9 +1873,21 @@ class PyCurlFileObject():
@@ -1582,9 +1883,21 @@ class PyCurlFileObject():
self.opts.progress_obj.end(self._amount_read)
self.fo.close()
@ -1225,7 +1245,7 @@ index e090e90..daa478d 100644
#####################################################################
# DEPRECATED FUNCTIONS
@@ -1621,6 +1924,460 @@ def retrygrab(url, filename=None, copy_local=0, close_connection=0,
@@ -1621,6 +1934,466 @@ def retrygrab(url, filename=None, copy_local=0, close_connection=0,
#####################################################################
@ -1378,7 +1398,7 @@ index e090e90..daa478d 100644
+ if DEBUG: DEBUG.info('success')
+ else:
+ ug_err = URLGrabError(int(line[4]), line[5])
+ if DEBUG: DEBUG.info('failure: %s', err)
+ if DEBUG: DEBUG.info('failure: %s', ug_err)
+ _TH.update(opts.url, int(line[2]), float(line[3]), ug_err, opts.async[0])
+ ret.append((opts, size, ug_err))
+ return ret
@ -1474,10 +1494,19 @@ index e090e90..daa478d 100644
+ for opts, size, ug_err in dl.perform():
+ key, limit = opts.async
+ host_con[key] -= 1
+
+ if ug_err is None:
+ if opts.checkfunc:
+ try: _run_callback(opts.checkfunc, opts)
+ except URLGrabError, ug_err: pass
+
+ if opts.progress_obj:
+ if opts.multi_progress_obj:
+ opts.multi_progress_obj.re.total += size - opts.size # correct totals
+ opts._progress.end(size)
+ if ug_err:
+ opts._progress.failure(None)
+ else:
+ opts.multi_progress_obj.re.total += size - opts.size # correct totals
+ opts._progress.end(size)
+ opts.multi_progress_obj.removeMeter(opts._progress)
+ else:
+ opts.progress_obj.start(text=opts.text, now=opts._progress)
@ -1486,11 +1515,7 @@ index e090e90..daa478d 100644
+ del opts._progress
+
+ if ug_err is None:
+ if opts.checkfunc:
+ try: _run_callback(opts.checkfunc, opts)
+ except URLGrabError, ug_err: pass
+ if ug_err is None:
+ continue
+ continue
+
+ retry = opts.retry or 0
+ if opts.failure_callback:
@ -1558,8 +1583,9 @@ index e090e90..daa478d 100644
+ speed = _TH.estimate(key)
+ speed /= 1 + host_con.get(key, 0)
+
+ # 2-tuple to select mirror with least failures
+ speed = -failed.get(key, 0), speed
+ # order by: least failures, private flag, best speed
+ private = mirror.get('kwargs', {}).get('private', False)
+ speed = -failed.get(key, 0), private, speed
+ if best is None or speed > best_speed:
+ best = mirror
+ best_speed = speed
@ -1831,9 +1857,20 @@ index dad410b..b17be17 100644
def urlopen(self, url, **kwargs):
kw = dict(kwargs)
diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py
index dd07c6a..ad57dbc 100644
index dd07c6a..077fd99 100644
--- a/urlgrabber/progress.py
+++ b/urlgrabber/progress.py
@@ -133,8 +133,8 @@ class BaseMeter:
# for a real gui, you probably want to override and put a call
# to your mainloop iteration function here
if now is None: now = time.time()
- if (now >= self.last_update_time + self.update_period) or \
- not self.last_update_time:
+ if (not self.last_update_time or
+ (now >= self.last_update_time + self.update_period)):
self.re.update(amount_read, now)
self.last_amount_read = amount_read
self.last_update_time = now
@@ -211,6 +211,21 @@ def text_meter_total_size(size, downloaded=0):
# 4. + ( 5, total: 32)
#
@ -1856,7 +1893,38 @@ index dd07c6a..ad57dbc 100644
class TextMeter(BaseMeter):
def __init__(self, fo=sys.stderr):
BaseMeter.__init__(self)
@@ -259,13 +274,10 @@ class TextMeter(BaseMeter):
@@ -218,7 +233,6 @@ class TextMeter(BaseMeter):
def _do_update(self, amount_read, now=None):
etime = self.re.elapsed_time()
- fetime = format_time(etime)
fread = format_number(amount_read)
#self.size = None
if self.text is not None:
@@ -234,16 +248,20 @@ class TextMeter(BaseMeter):
# Include text + ui_rate in minimal
tl = TerminalLine(8, 8+1+8)
+ if tl._llen > 80:
+ use_hours = True # For big screens, make it more readable.
+ else:
+ use_hours = False
ui_size = tl.add(' | %5sB' % fread)
if self.size is None:
- ui_time = tl.add(' %9s' % fetime)
+ ui_time = tl.add(' %9s' % format_time(etime, use_hours))
ui_end = tl.add(' ' * 5)
ui_rate = tl.add(' %5sB/s' % ave_dl)
out = '%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
ui_rate, ui_size, ui_time, ui_end)
else:
rtime = self.re.remaining_time()
- frtime = format_time(rtime)
+ frtime = format_time(rtime, use_hours)
frac = self.re.fraction_read()
ui_time = tl.add(' %9s' % frtime)
@@ -259,13 +277,10 @@ class TextMeter(BaseMeter):
ui_rate = tl.add(' %5sB/s' % ave_dl)
# Make text grow a bit before we start growing the bar too
blen = 4 + tl.rest_split(8 + 8 + 4)
@ -1874,21 +1942,36 @@ index dd07c6a..ad57dbc 100644
self.fo.write(out)
self.fo.flush()
@@ -284,12 +296,7 @@ class TextMeter(BaseMeter):
@@ -274,7 +289,6 @@ class TextMeter(BaseMeter):
global _text_meter_total_size
global _text_meter_sofar_size
- total_time = format_time(self.re.elapsed_time())
total_size = format_number(amount_read)
if self.text is not None:
text = self.text
@@ -282,14 +296,13 @@ class TextMeter(BaseMeter):
text = self.basename
tl = TerminalLine(8)
ui_size = tl.add(' | %5sB' % total_size)
ui_time = tl.add(' %9s' % total_time)
- ui_size = tl.add(' | %5sB' % total_size)
- ui_time = tl.add(' %9s' % total_time)
- not_done = self.size is not None and amount_read != self.size
- if not_done:
- ui_end = tl.add(' ... ')
- else:
+ if tl._llen > 80:
+ use_hours = True # For big screens, make it more readable.
else:
- ui_end = tl.add(' ' * 5)
-
+ use_hours = False
+ ui_size = tl.add(' | %5sB' % total_size)
+ ui_time = tl.add(' %9s' % format_time(self.re.elapsed_time(),use_hours))
+ ui_end, not_done = _term_add_end(tl, self.size, amount_read)
out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text,
ui_size, ui_time, ui_end)
self.fo.write(out)
@@ -331,12 +338,21 @@ class MultiFileHelper(BaseMeter):
@@ -331,12 +344,21 @@ class MultiFileHelper(BaseMeter):
def message(self, message):
self.master.message_meter(self, message)
@ -1912,7 +1995,7 @@ index dd07c6a..ad57dbc 100644
self.update_period = 0.3 # seconds
self.numfiles = None
@@ -369,6 +385,7 @@ class MultiFileMeter:
@@ -369,6 +391,7 @@ class MultiFileMeter:
def end(self, now=None):
if now is None: now = time.time()
@ -1920,7 +2003,18 @@ index dd07c6a..ad57dbc 100644
self._do_end(now)
def _do_end(self, now):
@@ -466,11 +483,21 @@ class MultiFileMeter:
@@ -407,8 +430,8 @@ class MultiFileMeter:
def update_meter(self, meter, now):
if not meter in self.meters:
raise ValueError('attempt to use orphaned meter')
- if (now >= self.last_update_time + self.update_period) or \
- not self.last_update_time:
+ if (not self.last_update_time or
+ (now >= self.last_update_time + self.update_period)):
self.re.update(self._amount_read(), now)
self.last_update_time = now
self._do_update_meter(meter, now)
@@ -466,34 +489,87 @@ class MultiFileMeter:
class TextMultiFileMeter(MultiFileMeter):
@ -1933,10 +2027,15 @@ index dd07c6a..ad57dbc 100644
# files: ###/### ###% data: ######/###### ###% time: ##:##:##/##:##:##
+# New output, like TextMeter output...
+# update: No size (minimal: 17 chars)
+# -----------------------------------
+# (<#file>/<#tot files>): <text> <rate> | <current size> <elapsed>
+# 8-48 1 8 3 6 1 7-9 5
+#
+# update: Size, All files
+# -----------------------
+# (<#file>/<#tot files>): <text> <pc> <bar> <rate> | <size> <eta time> ETA
+# 8-22 1 3-4 1 6-12 1 8 3 6 1 9 1 3 1
+# 8-22 1 3-4 1 6-12 1 8 3 6 1 7-9 1 3 1
+# end
+# ---
+# <text> | <file size> <file elapsed time>
@ -1944,24 +2043,31 @@ index dd07c6a..ad57dbc 100644
def _do_update_meter(self, meter, now):
self._lock.acquire()
try:
@@ -480,7 +507,7 @@ class TextMultiFileMeter(MultiFileMeter):
- format = "files: %3i/%-3i %3i%% data: %6.6s/%-6.6s %3i%% " \
- "time: %8.8s/%8.8s"
df = self.finished_files
tf = self.numfiles or 1
pf = 100 * float(df)/tf + 0.49
- pf = 100 * float(df)/tf + 0.49
+ # Don't use "percent of files complete" ...
+ # pf = 100 * float(df)/tf + 0.49
dd = self.re.last_amount_read
- td = self.total_size
+ td = self.re.total
pd = 100 * (self.re.fraction_read() or 0) + 0.49
dt = self.re.elapsed_time()
rt = self.re.remaining_time()
@@ -491,9 +518,41 @@ class TextMultiFileMeter(MultiFileMeter):
ftd = format_number(td) + 'B'
fdt = format_time(dt, 1)
ftt = format_time(tt, 1)
- if rt is None: tt = None
- else: tt = dt + rt
- fdd = format_number(dd) + 'B'
- ftd = format_number(td) + 'B'
- fdt = format_time(dt, 1)
- ftt = format_time(tt, 1)
-
- out = '%-79.79s' % (format % (df, tf, pf, fdd, ftd, pd, fdt, ftt))
- self.fo.write('\r' + out)
+
+ frac = self.re.fraction_read() or 0
+ pf = 100 * frac
+ ave_dl = format_number(self.re.average_rate())
+
+ # cycle through active meters
@ -1977,28 +2083,41 @@ index dd07c6a..ad57dbc 100644
+
+ # Include text + ui_rate in minimal
+ tl = TerminalLine(8, 8+1+8)
+ if tl._llen > 80:
+ use_hours = True # For big screens, make it more readable.
+ time_len = 9
+ else:
+ use_hours = False
+ time_len = 7
+
+ ui_size = tl.add(' | %5sB' % format_number(dd))
+
+ ui_time = tl.add(' %9s' % format_time(rt))
+ ui_end = tl.add(' ETA ')
+
+ ui_sofar_pc = tl.add(' %i%%' % pf,
+ full_len=len(" (100%)"))
+ ui_rate = tl.add(' %5sB/s' % ave_dl)
+
+ # Make text grow a bit before we start growing the bar too
+ blen = 4 + tl.rest_split(8 + 8 + 4)
+ ui_bar = _term_add_bar(tl, blen, frac)
+ out = '\r%-*.*s%s%s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
+ ui_sofar_pc, ui_bar,
+ ui_rate, ui_size, ui_time,
+ ui_end)
+ if not self.re.total:
+ ui_time = tl.add(' %*s' % (time_len,format_time(dt, use_hours)))
+ ui_end = tl.add(' ' * 5)
+ ui_rate = tl.add(' %5sB/s' % ave_dl)
+ out = '\r%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
+ ui_rate, ui_size, ui_time, ui_end)
+ else:
+ ui_time = tl.add(' %*s' % (time_len,format_time(rt, use_hours)))
+ ui_end = tl.add(' ETA ')
+
+ ui_sofar_pc = tl.add(' %i%%' % pf,
+ full_len=len(" (100%)"))
+ ui_rate = tl.add(' %5sB/s' % ave_dl)
+
+ # Make text grow a bit before we start growing the bar too
+ blen = 4 + tl.rest_split(8 + 8 + 4)
+ ui_bar = _term_add_bar(tl, blen, frac)
+ out = '\r%-*.*s%s%s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
+ ui_sofar_pc, ui_bar,
+ ui_rate, ui_size, ui_time,
+ ui_end)
+ self.fo.write(out)
self.fo.flush()
finally:
self._lock.release()
@@ -502,18 +561,30 @@ class TextMultiFileMeter(MultiFileMeter):
@@ -502,24 +578,40 @@ class TextMultiFileMeter(MultiFileMeter):
self._lock.acquire()
try:
format = "%-30.30s %6.6s %8.8s %9.9s"
@ -2007,7 +2126,7 @@ index dd07c6a..ad57dbc 100644
size = meter.last_amount_read
fsize = format_number(size) + 'B'
et = meter.re.elapsed_time()
fet = format_time(et, 1)
- fet = format_time(et, 1)
- frate = format_number(size / et) + 'B/s'
-
- out = '%-79.79s' % (format % (fn, fsize, fet, frate))
@ -2016,15 +2135,20 @@ index dd07c6a..ad57dbc 100644
+ df = self.finished_files
+ tf = self.numfiles or 1
+
+ total_time = format_time(et)
+ total_size = format_number(size)
+ text = meter.text or meter.basename
+ if tf > 1:
+ text = '(%u/%u): %s' % (df, tf, text)
+
+ tl = TerminalLine(8)
+ if tl._llen > 80:
+ use_hours = True # For big screens, make it more readable.
+ time_len = 9
+ else:
+ use_hours = False
+ time_len = 7
+ ui_size = tl.add(' | %5sB' % total_size)
+ ui_time = tl.add(' %9s' % total_time)
+ ui_time = tl.add(' %*s' % (time_len, format_time(et, use_hours)))
+ ui_end, not_done = _term_add_end(tl, meter.size, size)
+ out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text,
+ ui_size, ui_time, ui_end)
@ -2035,7 +2159,14 @@ index dd07c6a..ad57dbc 100644
def _do_failure_meter(self, meter, message, now):
self._lock.acquire()
@@ -536,15 +607,6 @@ class TextMultiFileMeter(MultiFileMeter):
try:
format = "%-30.30s %6.6s %s"
- fn = meter.basename
+ fn = meter.text or meter.basename
if type(message) in (type(''), type(u'')):
message = message.splitlines()
if not message: message = ['']
@@ -536,15 +628,6 @@ class TextMultiFileMeter(MultiFileMeter):
pass
finally:
self._lock.release()
@ -2051,7 +2182,7 @@ index dd07c6a..ad57dbc 100644
######################################################################
# support classes and functions
@@ -658,6 +720,8 @@ def format_time(seconds, use_hours=0):
@@ -658,6 +741,8 @@ def format_time(seconds, use_hours=0):
if seconds is None or seconds < 0:
if use_hours: return '--:--:--'
else: return '--:--'
@ -2060,3 +2191,81 @@ index dd07c6a..ad57dbc 100644
else:
seconds = int(seconds)
minutes = seconds / 60
@@ -722,9 +807,77 @@ def _tst(fn, cur, tot, beg, size, *args):
time.sleep(delay)
tm.end(size)
+def _mtst(datas, *args):
+ print '-' * 79
+ tm = TextMultiFileMeter(threaded=False)
+
+ dl_sizes = {}
+
+ num = 0
+ total_size = 0
+ dl_total_size = 0
+ for data in datas:
+ dl_size = None
+ if len(data) == 2:
+ fn, size = data
+ dl_size = size
+ if len(data) == 3:
+ fn, size, dl_size = data
+ nm = tm.newMeter()
+ nm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size,
+ text=fn)
+ num += 1
+ assert dl_size is not None
+ dl_total_size += dl_size
+ dl_sizes[nm] = dl_size
+ if size is None or total_size is None:
+ total_size = None
+ else:
+ total_size += size
+ tm.start(num, total_size)
+
+ num = 0
+ off = 0
+ for (inc, delay) in args:
+ off += 1
+ while num < ((dl_total_size * off) / len(args)):
+ num += inc
+ for nm in tm.meters[:]:
+ if dl_sizes[nm] <= num:
+ nm.end(dl_sizes[nm])
+ tm.removeMeter(nm)
+ else:
+ nm.update(num)
+ time.sleep(delay)
+ assert not tm.meters
+
if __name__ == "__main__":
# (1/2): subversion-1.4.4-7.x86_64.rpm 2.4 MB / 85 kB/s 00:28
# (2/2): mercurial-0.9.5-6.fc8.x86_64.rpm 924 kB / 106 kB/s 00:08
+ if len(sys.argv) >= 2 and sys.argv[1] == 'multi':
+ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
+ ("s-1.0.1-1.fc8.i386.rpm", 5000),
+ ("m-1.0.1-2.fc8.i386.rpm", 10000)),
+ (100, 0.33), (500, 0.25), (1000, 0.1))
+
+ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
+ ("s-1.0.1-1.fc8.i386.rpm", 5000),
+ ("m-1.0.1-2.fc8.i386.rpm", None, 10000)),
+ (100, 0.33), (500, 0.25), (1000, 0.1))
+
+ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
+ ("s-1.0.1-1.fc8.i386.rpm", 2500000),
+ ("m-1.0.1-2.fc8.i386.rpm", 10000)),
+ (10, 0.2), (50, 0.1), (1000, 0.1))
+
+ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
+ ("s-1.0.1-1.fc8.i386.rpm", None, 2500000),
+ ("m-1.0.1-2.fc8.i386.rpm", None, 10000)),
+ (10, 0.2), (50, 0.1), (1000, 0.1))
+ # (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
+ # (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
+ sys.exit(0)
+
if len(sys.argv) >= 2 and sys.argv[1] == 'total':
text_meter_total_size(1000 + 10000 + 10000 + 1000000 + 1000000 +
1000000 + 10000 + 10000 + 10000 + 1000000)

Write Preview
Loading…
Cancel
Save