diff --git a/python-urlgrabber.spec b/python-urlgrabber.spec index c33f014..2975737 100644 --- a/python-urlgrabber.spec +++ b/python-urlgrabber.spec @@ -3,7 +3,7 @@ Summary: A high-level cross-protocol url-grabber Name: python-urlgrabber Version: 3.9.1 -Release: 19%{?dist} +Release: 20%{?dist} Source0: urlgrabber-%{version}.tar.gz Patch1: urlgrabber-HEAD.patch @@ -44,6 +44,10 @@ rm -rf $RPM_BUILD_ROOT %attr(0755,root,root) %{_libexecdir}/urlgrabber-ext-down %changelog +* Tue Sep 4 2012 Zdeněk Pavlas - 3.9.1-20 +- Update to latest HEAD. +- Fixed BZ 851178, 854075. + * Mon Aug 27 2012 Zdeněk Pavlas - 3.9.1-19 - timedhosts: defer 1st update until a 1MB+ download. BZ 851178 diff --git a/urlgrabber-HEAD.patch b/urlgrabber-HEAD.patch index a092d3a..ef304ad 100644 --- a/urlgrabber-HEAD.patch +++ b/urlgrabber-HEAD.patch @@ -73,10 +73,10 @@ index 518e512..09cd896 100644 print __doc__ diff --git a/scripts/urlgrabber-ext-down b/scripts/urlgrabber-ext-down new file mode 100755 -index 0000000..3da55a4 +index 0000000..3dafb12 --- /dev/null +++ b/scripts/urlgrabber-ext-down -@@ -0,0 +1,72 @@ +@@ -0,0 +1,75 @@ +#! /usr/bin/python +# A very simple external downloader +# Copyright 2011-2012 Zdenek Pavlas @@ -134,18 +134,21 @@ index 0000000..3da55a4 + if opts.progress_obj: + opts.progress_obj = ProxyProgress() + opts.progress_obj._id = cnt -+ tm = time.time() ++ ++ dlsz = dltm = 0 + try: + fo = PyCurlFileObject(opts.url, opts.filename, opts) + fo._do_grab() + fo.fo.close() + size = fo._amount_read -+ dlsz = size - fo._reget_length ++ if fo._tm_last: ++ dlsz = fo._tm_last[0] - fo._tm_first[0] ++ dltm = fo._tm_last[1] - fo._tm_first[1] + ug_err = 'OK' + except URLGrabError, e: -+ size = dlsz = 0 ++ size = 0 + ug_err = '%d %s' % e.args -+ write('%d %d %d %.3f %s\n', opts._id, size, dlsz, time.time() - tm, ug_err) ++ write('%d %d %d %.3f %s\n', opts._id, size, dlsz, dltm, ug_err) + +if __name__ == '__main__': + main() @@ -233,7 +236,7 @@ index 3e5f3b7..8eeaeda 100644 return (fb,lb) diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py -index e090e90..daa478d 100644 +index e090e90..01218b0 100644 --- a/urlgrabber/grabber.py +++ b/urlgrabber/grabber.py @@ -49,11 +49,26 @@ GENERAL ARGUMENTS (kwargs) @@ -678,7 +681,7 @@ index e090e90..daa478d 100644 if scheme == 'file' and not opts.copy_local: # just return the name of the local file - don't make a # copy currently -@@ -950,41 +1114,49 @@ class URLGrabber: +@@ -950,41 +1114,51 @@ class URLGrabber: elif not opts.range: if not opts.checkfunc is None: @@ -700,11 +703,13 @@ index e090e90..daa478d 100644 + return filename + def retryfunc(opts, url, filename): -+ tm = time.time() fo = PyCurlFileObject(url, filename, opts) try: fo._do_grab() -+ _TH.update(url, fo._amount_read - fo._reget_length, time.time() - tm, None) ++ if fo._tm_last: ++ dlsz = fo._tm_last[0] - fo._tm_first[0] ++ dltm = fo._tm_last[1] - fo._tm_first[1] ++ _TH.update(url, dlsz, dltm, None) if not opts.checkfunc is None: - cb_func, cb_args, cb_kwargs = \ - self._make_callback(opts.checkfunc) @@ -743,7 +748,7 @@ index e090e90..daa478d 100644 if limit is not None: limit = limit + 1 -@@ -1000,12 +1172,8 @@ class URLGrabber: +@@ -1000,12 +1174,8 @@ class URLGrabber: else: s = fo.read(limit) if not opts.checkfunc is None: @@ -758,7 +763,7 @@ index e090e90..daa478d 100644 finally: fo.close() return s -@@ -1020,6 +1188,7 @@ class URLGrabber: +@@ -1020,6 +1190,7 @@ class URLGrabber: return s def _make_callback(self, callback_obj): @@ -766,7 +771,7 @@ index e090e90..daa478d 100644 if callable(callback_obj): return callback_obj, (), {} else: -@@ -1030,7 +1199,7 @@ class URLGrabber: +@@ -1030,7 +1201,7 @@ class URLGrabber: default_grabber = URLGrabber() @@ -775,13 +780,15 @@ index e090e90..daa478d 100644 def __init__(self, url, filename, opts): self.fo = None self._hdr_dump = '' -@@ -1052,10 +1221,11 @@ class PyCurlFileObject(): +@@ -1052,10 +1223,13 @@ class PyCurlFileObject(): self._reget_length = 0 self._prog_running = False self._error = (None, None) - self.size = None + self.size = 0 + self._hdr_ended = False ++ self._tm_first = None ++ self._tm_last = None self._do_open() - @@ -789,7 +796,20 @@ index e090e90..daa478d 100644 def __getattr__(self, name): """This effectively allows us to wrap at the instance level. Any attribute not found in _this_ object will be searched for -@@ -1085,9 +1255,14 @@ class PyCurlFileObject(): +@@ -1067,6 +1241,12 @@ class PyCurlFileObject(): + + def _retrieve(self, buf): + try: ++ tm = self._amount_read + len(buf), time.time() ++ if self._tm_first is None: ++ self._tm_first = tm ++ else: ++ self._tm_last = tm ++ + if not self._prog_running: + if self.opts.progress_obj: + size = self.size + self._reget_length +@@ -1085,9 +1265,14 @@ class PyCurlFileObject(): return -1 def _hdr_retrieve(self, buf): @@ -805,7 +825,7 @@ index e090e90..daa478d 100644 try: self._hdr_dump += buf # we have to get the size before we do the progress obj start -@@ -1104,7 +1279,17 @@ class PyCurlFileObject(): +@@ -1104,7 +1289,17 @@ class PyCurlFileObject(): s = parse150(buf) if s: self.size = int(s) @@ -824,7 +844,7 @@ index e090e90..daa478d 100644 return len(buf) except KeyboardInterrupt: return pycurl.READFUNC_ABORT -@@ -1113,8 +1298,10 @@ class PyCurlFileObject(): +@@ -1113,8 +1308,10 @@ class PyCurlFileObject(): if self._parsed_hdr: return self._parsed_hdr statusend = self._hdr_dump.find('\n') @@ -835,7 +855,7 @@ index e090e90..daa478d 100644 self._parsed_hdr = mimetools.Message(hdrfp) return self._parsed_hdr -@@ -1127,6 +1314,9 @@ class PyCurlFileObject(): +@@ -1127,6 +1324,9 @@ class PyCurlFileObject(): if not opts: opts = self.opts @@ -845,7 +865,7 @@ index e090e90..daa478d 100644 # defaults we're always going to set self.curl_obj.setopt(pycurl.NOPROGRESS, False) -@@ -1136,11 +1326,21 @@ class PyCurlFileObject(): +@@ -1136,11 +1336,21 @@ class PyCurlFileObject(): self.curl_obj.setopt(pycurl.PROGRESSFUNCTION, self._progress_update) self.curl_obj.setopt(pycurl.FAILONERROR, True) self.curl_obj.setopt(pycurl.OPT_FILETIME, True) @@ -867,7 +887,7 @@ index e090e90..daa478d 100644 # maybe to be options later self.curl_obj.setopt(pycurl.FOLLOWLOCATION, True) -@@ -1148,9 +1348,11 @@ class PyCurlFileObject(): +@@ -1148,9 +1358,11 @@ class PyCurlFileObject(): # timeouts timeout = 300 @@ -882,7 +902,7 @@ index e090e90..daa478d 100644 # ssl options if self.scheme == 'https': -@@ -1158,13 +1360,16 @@ class PyCurlFileObject(): +@@ -1158,13 +1370,16 @@ class PyCurlFileObject(): self.curl_obj.setopt(pycurl.CAPATH, opts.ssl_ca_cert) self.curl_obj.setopt(pycurl.CAINFO, opts.ssl_ca_cert) self.curl_obj.setopt(pycurl.SSL_VERIFYPEER, opts.ssl_verify_peer) @@ -900,7 +920,7 @@ index e090e90..daa478d 100644 if opts.ssl_cert_type: self.curl_obj.setopt(pycurl.SSLCERTTYPE, opts.ssl_cert_type) if opts.ssl_key_pass: -@@ -1187,28 +1392,26 @@ class PyCurlFileObject(): +@@ -1187,28 +1402,26 @@ class PyCurlFileObject(): if hasattr(opts, 'raw_throttle') and opts.raw_throttle(): self.curl_obj.setopt(pycurl.MAX_RECV_SPEED_LARGE, int(opts.raw_throttle())) @@ -945,7 +965,7 @@ index e090e90..daa478d 100644 # our url self.curl_obj.setopt(pycurl.URL, self.url) -@@ -1228,12 +1431,14 @@ class PyCurlFileObject(): +@@ -1228,12 +1441,14 @@ class PyCurlFileObject(): code = self.http_code errcode = e.args[0] @@ -962,7 +982,7 @@ index e090e90..daa478d 100644 # this is probably wrong but ultimately this is what happens # we have a legit http code and a pycurl 'writer failed' code -@@ -1244,23 +1449,23 @@ class PyCurlFileObject(): +@@ -1244,23 +1459,23 @@ class PyCurlFileObject(): raise KeyboardInterrupt elif errcode == 28: @@ -993,7 +1013,7 @@ index e090e90..daa478d 100644 # this is probably wrong but ultimately this is what happens # we have a legit http code and a pycurl 'writer failed' code # which almost always means something aborted it from outside -@@ -1272,33 +1477,94 @@ class PyCurlFileObject(): +@@ -1272,33 +1487,94 @@ class PyCurlFileObject(): elif errcode == 58: msg = _("problem with the local client certificate") err = URLGrabError(14, msg) @@ -1095,7 +1115,7 @@ index e090e90..daa478d 100644 def _do_open(self): self.curl_obj = _curl_cache -@@ -1333,7 +1599,11 @@ class PyCurlFileObject(): +@@ -1333,7 +1609,11 @@ class PyCurlFileObject(): if self.opts.range: rt = self.opts.range @@ -1108,7 +1128,7 @@ index e090e90..daa478d 100644 if rt: header = range_tuple_to_header(rt) -@@ -1434,21 +1704,46 @@ class PyCurlFileObject(): +@@ -1434,21 +1714,46 @@ class PyCurlFileObject(): #fh, self._temp_name = mkstemp() #self.fo = open(self._temp_name, 'wb') @@ -1162,7 +1182,7 @@ index e090e90..daa478d 100644 else: #self.fo = open(self._temp_name, 'r') self.fo.seek(0) -@@ -1526,17 +1821,20 @@ class PyCurlFileObject(): +@@ -1526,17 +1831,20 @@ class PyCurlFileObject(): if self._prog_running: downloaded += self._reget_length self.opts.progress_obj.update(downloaded) @@ -1188,7 +1208,7 @@ index e090e90..daa478d 100644 msg = _("Downloaded more than max size for %s: %s > %s") \ % (self.url, cur, max_size) -@@ -1544,13 +1842,6 @@ class PyCurlFileObject(): +@@ -1544,13 +1852,6 @@ class PyCurlFileObject(): return True return False @@ -1202,7 +1222,7 @@ index e090e90..daa478d 100644 def read(self, amt=None): self._fill_buffer(amt) if amt is None: -@@ -1582,9 +1873,21 @@ class PyCurlFileObject(): +@@ -1582,9 +1883,21 @@ class PyCurlFileObject(): self.opts.progress_obj.end(self._amount_read) self.fo.close() @@ -1225,7 +1245,7 @@ index e090e90..daa478d 100644 ##################################################################### # DEPRECATED FUNCTIONS -@@ -1621,6 +1924,460 @@ def retrygrab(url, filename=None, copy_local=0, close_connection=0, +@@ -1621,6 +1934,466 @@ def retrygrab(url, filename=None, copy_local=0, close_connection=0, ##################################################################### @@ -1378,7 +1398,7 @@ index e090e90..daa478d 100644 + if DEBUG: DEBUG.info('success') + else: + ug_err = URLGrabError(int(line[4]), line[5]) -+ if DEBUG: DEBUG.info('failure: %s', err) ++ if DEBUG: DEBUG.info('failure: %s', ug_err) + _TH.update(opts.url, int(line[2]), float(line[3]), ug_err, opts.async[0]) + ret.append((opts, size, ug_err)) + return ret @@ -1474,10 +1494,19 @@ index e090e90..daa478d 100644 + for opts, size, ug_err in dl.perform(): + key, limit = opts.async + host_con[key] -= 1 ++ ++ if ug_err is None: ++ if opts.checkfunc: ++ try: _run_callback(opts.checkfunc, opts) ++ except URLGrabError, ug_err: pass ++ + if opts.progress_obj: + if opts.multi_progress_obj: -+ opts.multi_progress_obj.re.total += size - opts.size # correct totals -+ opts._progress.end(size) ++ if ug_err: ++ opts._progress.failure(None) ++ else: ++ opts.multi_progress_obj.re.total += size - opts.size # correct totals ++ opts._progress.end(size) + opts.multi_progress_obj.removeMeter(opts._progress) + else: + opts.progress_obj.start(text=opts.text, now=opts._progress) @@ -1486,11 +1515,7 @@ index e090e90..daa478d 100644 + del opts._progress + + if ug_err is None: -+ if opts.checkfunc: -+ try: _run_callback(opts.checkfunc, opts) -+ except URLGrabError, ug_err: pass -+ if ug_err is None: -+ continue ++ continue + + retry = opts.retry or 0 + if opts.failure_callback: @@ -1558,8 +1583,9 @@ index e090e90..daa478d 100644 + speed = _TH.estimate(key) + speed /= 1 + host_con.get(key, 0) + -+ # 2-tuple to select mirror with least failures -+ speed = -failed.get(key, 0), speed ++ # order by: least failures, private flag, best speed ++ private = mirror.get('kwargs', {}).get('private', False) ++ speed = -failed.get(key, 0), private, speed + if best is None or speed > best_speed: + best = mirror + best_speed = speed @@ -1831,9 +1857,20 @@ index dad410b..b17be17 100644 def urlopen(self, url, **kwargs): kw = dict(kwargs) diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py -index dd07c6a..ad57dbc 100644 +index dd07c6a..077fd99 100644 --- a/urlgrabber/progress.py +++ b/urlgrabber/progress.py +@@ -133,8 +133,8 @@ class BaseMeter: + # for a real gui, you probably want to override and put a call + # to your mainloop iteration function here + if now is None: now = time.time() +- if (now >= self.last_update_time + self.update_period) or \ +- not self.last_update_time: ++ if (not self.last_update_time or ++ (now >= self.last_update_time + self.update_period)): + self.re.update(amount_read, now) + self.last_amount_read = amount_read + self.last_update_time = now @@ -211,6 +211,21 @@ def text_meter_total_size(size, downloaded=0): # 4. + ( 5, total: 32) # @@ -1856,7 +1893,38 @@ index dd07c6a..ad57dbc 100644 class TextMeter(BaseMeter): def __init__(self, fo=sys.stderr): BaseMeter.__init__(self) -@@ -259,13 +274,10 @@ class TextMeter(BaseMeter): +@@ -218,7 +233,6 @@ class TextMeter(BaseMeter): + + def _do_update(self, amount_read, now=None): + etime = self.re.elapsed_time() +- fetime = format_time(etime) + fread = format_number(amount_read) + #self.size = None + if self.text is not None: +@@ -234,16 +248,20 @@ class TextMeter(BaseMeter): + + # Include text + ui_rate in minimal + tl = TerminalLine(8, 8+1+8) ++ if tl._llen > 80: ++ use_hours = True # For big screens, make it more readable. ++ else: ++ use_hours = False + ui_size = tl.add(' | %5sB' % fread) + if self.size is None: +- ui_time = tl.add(' %9s' % fetime) ++ ui_time = tl.add(' %9s' % format_time(etime, use_hours)) + ui_end = tl.add(' ' * 5) + ui_rate = tl.add(' %5sB/s' % ave_dl) + out = '%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text, + ui_rate, ui_size, ui_time, ui_end) + else: + rtime = self.re.remaining_time() +- frtime = format_time(rtime) ++ frtime = format_time(rtime, use_hours) + frac = self.re.fraction_read() + + ui_time = tl.add(' %9s' % frtime) +@@ -259,13 +277,10 @@ class TextMeter(BaseMeter): ui_rate = tl.add(' %5sB/s' % ave_dl) # Make text grow a bit before we start growing the bar too blen = 4 + tl.rest_split(8 + 8 + 4) @@ -1874,21 +1942,36 @@ index dd07c6a..ad57dbc 100644 self.fo.write(out) self.fo.flush() -@@ -284,12 +296,7 @@ class TextMeter(BaseMeter): +@@ -274,7 +289,6 @@ class TextMeter(BaseMeter): + global _text_meter_total_size + global _text_meter_sofar_size + +- total_time = format_time(self.re.elapsed_time()) + total_size = format_number(amount_read) + if self.text is not None: + text = self.text +@@ -282,14 +296,13 @@ class TextMeter(BaseMeter): + text = self.basename + tl = TerminalLine(8) - ui_size = tl.add(' | %5sB' % total_size) - ui_time = tl.add(' %9s' % total_time) +- ui_size = tl.add(' | %5sB' % total_size) +- ui_time = tl.add(' %9s' % total_time) - not_done = self.size is not None and amount_read != self.size - if not_done: - ui_end = tl.add(' ... ') -- else: ++ if tl._llen > 80: ++ use_hours = True # For big screens, make it more readable. + else: - ui_end = tl.add(' ' * 5) - ++ use_hours = False ++ ui_size = tl.add(' | %5sB' % total_size) ++ ui_time = tl.add(' %9s' % format_time(self.re.elapsed_time(),use_hours)) + ui_end, not_done = _term_add_end(tl, self.size, amount_read) out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text, ui_size, ui_time, ui_end) self.fo.write(out) -@@ -331,12 +338,21 @@ class MultiFileHelper(BaseMeter): +@@ -331,12 +344,21 @@ class MultiFileHelper(BaseMeter): def message(self, message): self.master.message_meter(self, message) @@ -1912,7 +1995,7 @@ index dd07c6a..ad57dbc 100644 self.update_period = 0.3 # seconds self.numfiles = None -@@ -369,6 +385,7 @@ class MultiFileMeter: +@@ -369,6 +391,7 @@ class MultiFileMeter: def end(self, now=None): if now is None: now = time.time() @@ -1920,7 +2003,18 @@ index dd07c6a..ad57dbc 100644 self._do_end(now) def _do_end(self, now): -@@ -466,11 +483,21 @@ class MultiFileMeter: +@@ -407,8 +430,8 @@ class MultiFileMeter: + def update_meter(self, meter, now): + if not meter in self.meters: + raise ValueError('attempt to use orphaned meter') +- if (now >= self.last_update_time + self.update_period) or \ +- not self.last_update_time: ++ if (not self.last_update_time or ++ (now >= self.last_update_time + self.update_period)): + self.re.update(self._amount_read(), now) + self.last_update_time = now + self._do_update_meter(meter, now) +@@ -466,34 +489,87 @@ class MultiFileMeter: class TextMultiFileMeter(MultiFileMeter): @@ -1933,10 +2027,15 @@ index dd07c6a..ad57dbc 100644 # files: ###/### ###% data: ######/###### ###% time: ##:##:##/##:##:## +# New output, like TextMeter output... ++# update: No size (minimal: 17 chars) ++# ----------------------------------- ++# (<#file>/<#tot files>): | ++# 8-48 1 8 3 6 1 7-9 5 ++# +# update: Size, All files +# ----------------------- +# (<#file>/<#tot files>): | ETA -+# 8-22 1 3-4 1 6-12 1 8 3 6 1 9 1 3 1 ++# 8-22 1 3-4 1 6-12 1 8 3 6 1 7-9 1 3 1 +# end +# --- +# | @@ -1944,24 +2043,31 @@ index dd07c6a..ad57dbc 100644 def _do_update_meter(self, meter, now): self._lock.acquire() try: -@@ -480,7 +507,7 @@ class TextMultiFileMeter(MultiFileMeter): +- format = "files: %3i/%-3i %3i%% data: %6.6s/%-6.6s %3i%% " \ +- "time: %8.8s/%8.8s" + df = self.finished_files tf = self.numfiles or 1 - pf = 100 * float(df)/tf + 0.49 +- pf = 100 * float(df)/tf + 0.49 ++ # Don't use "percent of files complete" ... ++ # pf = 100 * float(df)/tf + 0.49 dd = self.re.last_amount_read - td = self.total_size + td = self.re.total pd = 100 * (self.re.fraction_read() or 0) + 0.49 dt = self.re.elapsed_time() rt = self.re.remaining_time() -@@ -491,9 +518,41 @@ class TextMultiFileMeter(MultiFileMeter): - ftd = format_number(td) + 'B' - fdt = format_time(dt, 1) - ftt = format_time(tt, 1) +- if rt is None: tt = None +- else: tt = dt + rt + +- fdd = format_number(dd) + 'B' +- ftd = format_number(td) + 'B' +- fdt = format_time(dt, 1) +- ftt = format_time(tt, 1) - - out = '%-79.79s' % (format % (df, tf, pf, fdd, ftd, pd, fdt, ftt)) - self.fo.write('\r' + out) -+ + frac = self.re.fraction_read() or 0 ++ pf = 100 * frac + ave_dl = format_number(self.re.average_rate()) + + # cycle through active meters @@ -1977,28 +2083,41 @@ index dd07c6a..ad57dbc 100644 + + # Include text + ui_rate in minimal + tl = TerminalLine(8, 8+1+8) ++ if tl._llen > 80: ++ use_hours = True # For big screens, make it more readable. ++ time_len = 9 ++ else: ++ use_hours = False ++ time_len = 7 + + ui_size = tl.add(' | %5sB' % format_number(dd)) + -+ ui_time = tl.add(' %9s' % format_time(rt)) -+ ui_end = tl.add(' ETA ') -+ -+ ui_sofar_pc = tl.add(' %i%%' % pf, -+ full_len=len(" (100%)")) -+ ui_rate = tl.add(' %5sB/s' % ave_dl) -+ -+ # Make text grow a bit before we start growing the bar too -+ blen = 4 + tl.rest_split(8 + 8 + 4) -+ ui_bar = _term_add_bar(tl, blen, frac) -+ out = '\r%-*.*s%s%s%s%s%s%s\r' % (tl.rest(), tl.rest(), text, -+ ui_sofar_pc, ui_bar, -+ ui_rate, ui_size, ui_time, -+ ui_end) ++ if not self.re.total: ++ ui_time = tl.add(' %*s' % (time_len,format_time(dt, use_hours))) ++ ui_end = tl.add(' ' * 5) ++ ui_rate = tl.add(' %5sB/s' % ave_dl) ++ out = '\r%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text, ++ ui_rate, ui_size, ui_time, ui_end) ++ else: ++ ui_time = tl.add(' %*s' % (time_len,format_time(rt, use_hours))) ++ ui_end = tl.add(' ETA ') ++ ++ ui_sofar_pc = tl.add(' %i%%' % pf, ++ full_len=len(" (100%)")) ++ ui_rate = tl.add(' %5sB/s' % ave_dl) ++ ++ # Make text grow a bit before we start growing the bar too ++ blen = 4 + tl.rest_split(8 + 8 + 4) ++ ui_bar = _term_add_bar(tl, blen, frac) ++ out = '\r%-*.*s%s%s%s%s%s%s\r' % (tl.rest(), tl.rest(), text, ++ ui_sofar_pc, ui_bar, ++ ui_rate, ui_size, ui_time, ++ ui_end) + self.fo.write(out) self.fo.flush() finally: self._lock.release() -@@ -502,18 +561,30 @@ class TextMultiFileMeter(MultiFileMeter): +@@ -502,24 +578,40 @@ class TextMultiFileMeter(MultiFileMeter): self._lock.acquire() try: format = "%-30.30s %6.6s %8.8s %9.9s" @@ -2007,7 +2126,7 @@ index dd07c6a..ad57dbc 100644 size = meter.last_amount_read fsize = format_number(size) + 'B' et = meter.re.elapsed_time() - fet = format_time(et, 1) +- fet = format_time(et, 1) - frate = format_number(size / et) + 'B/s' - - out = '%-79.79s' % (format % (fn, fsize, fet, frate)) @@ -2016,15 +2135,20 @@ index dd07c6a..ad57dbc 100644 + df = self.finished_files + tf = self.numfiles or 1 + -+ total_time = format_time(et) + total_size = format_number(size) + text = meter.text or meter.basename + if tf > 1: + text = '(%u/%u): %s' % (df, tf, text) + + tl = TerminalLine(8) ++ if tl._llen > 80: ++ use_hours = True # For big screens, make it more readable. ++ time_len = 9 ++ else: ++ use_hours = False ++ time_len = 7 + ui_size = tl.add(' | %5sB' % total_size) -+ ui_time = tl.add(' %9s' % total_time) ++ ui_time = tl.add(' %*s' % (time_len, format_time(et, use_hours))) + ui_end, not_done = _term_add_end(tl, meter.size, size) + out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text, + ui_size, ui_time, ui_end) @@ -2035,7 +2159,14 @@ index dd07c6a..ad57dbc 100644 def _do_failure_meter(self, meter, message, now): self._lock.acquire() -@@ -536,15 +607,6 @@ class TextMultiFileMeter(MultiFileMeter): + try: + format = "%-30.30s %6.6s %s" +- fn = meter.basename ++ fn = meter.text or meter.basename + if type(message) in (type(''), type(u'')): + message = message.splitlines() + if not message: message = [''] +@@ -536,15 +628,6 @@ class TextMultiFileMeter(MultiFileMeter): pass finally: self._lock.release() @@ -2051,7 +2182,7 @@ index dd07c6a..ad57dbc 100644 ###################################################################### # support classes and functions -@@ -658,6 +720,8 @@ def format_time(seconds, use_hours=0): +@@ -658,6 +741,8 @@ def format_time(seconds, use_hours=0): if seconds is None or seconds < 0: if use_hours: return '--:--:--' else: return '--:--' @@ -2060,3 +2191,81 @@ index dd07c6a..ad57dbc 100644 else: seconds = int(seconds) minutes = seconds / 60 +@@ -722,9 +807,77 @@ def _tst(fn, cur, tot, beg, size, *args): + time.sleep(delay) + tm.end(size) + ++def _mtst(datas, *args): ++ print '-' * 79 ++ tm = TextMultiFileMeter(threaded=False) ++ ++ dl_sizes = {} ++ ++ num = 0 ++ total_size = 0 ++ dl_total_size = 0 ++ for data in datas: ++ dl_size = None ++ if len(data) == 2: ++ fn, size = data ++ dl_size = size ++ if len(data) == 3: ++ fn, size, dl_size = data ++ nm = tm.newMeter() ++ nm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size, ++ text=fn) ++ num += 1 ++ assert dl_size is not None ++ dl_total_size += dl_size ++ dl_sizes[nm] = dl_size ++ if size is None or total_size is None: ++ total_size = None ++ else: ++ total_size += size ++ tm.start(num, total_size) ++ ++ num = 0 ++ off = 0 ++ for (inc, delay) in args: ++ off += 1 ++ while num < ((dl_total_size * off) / len(args)): ++ num += inc ++ for nm in tm.meters[:]: ++ if dl_sizes[nm] <= num: ++ nm.end(dl_sizes[nm]) ++ tm.removeMeter(nm) ++ else: ++ nm.update(num) ++ time.sleep(delay) ++ assert not tm.meters ++ + if __name__ == "__main__": + # (1/2): subversion-1.4.4-7.x86_64.rpm 2.4 MB / 85 kB/s 00:28 + # (2/2): mercurial-0.9.5-6.fc8.x86_64.rpm 924 kB / 106 kB/s 00:08 ++ if len(sys.argv) >= 2 and sys.argv[1] == 'multi': ++ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000), ++ ("s-1.0.1-1.fc8.i386.rpm", 5000), ++ ("m-1.0.1-2.fc8.i386.rpm", 10000)), ++ (100, 0.33), (500, 0.25), (1000, 0.1)) ++ ++ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000), ++ ("s-1.0.1-1.fc8.i386.rpm", 5000), ++ ("m-1.0.1-2.fc8.i386.rpm", None, 10000)), ++ (100, 0.33), (500, 0.25), (1000, 0.1)) ++ ++ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000), ++ ("s-1.0.1-1.fc8.i386.rpm", 2500000), ++ ("m-1.0.1-2.fc8.i386.rpm", 10000)), ++ (10, 0.2), (50, 0.1), (1000, 0.1)) ++ ++ _mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000), ++ ("s-1.0.1-1.fc8.i386.rpm", None, 2500000), ++ ("m-1.0.1-2.fc8.i386.rpm", None, 10000)), ++ (10, 0.2), (50, 0.1), (1000, 0.1)) ++ # (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25)) ++ # (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25)) ++ sys.exit(0) ++ + if len(sys.argv) >= 2 and sys.argv[1] == 'total': + text_meter_total_size(1000 + 10000 + 10000 + 1000000 + 1000000 + + 1000000 + 10000 + 10000 + 10000 + 1000000)