Index: urlgrabber/progress.py =================================================================== RCS file: /home/groups/urlgrabber/cvs-root/urlgrabber/urlgrabber/progress.py,v retrieving revision 1.7 diff -u -r1.7 urlgrabber/progress.py --- urlgrabber/progress.py 19 Aug 2005 21:59:07 -0000 1.7 +++ urlgrabber/progress.py 7 Apr 2008 16:33:30 -0000 @@ -83,6 +83,17 @@ def _do_end(self, amount_read, now=None): pass +# This is kind of a hack, but progress is gotten from grabber which doesn't +# know about the total size to download. So we do this so we can get the data +# out of band here. This will be "fixed" one way or anther soon. +_text_meter_total_size = 0 +_text_meter_sofar_size = 0 +def text_meter_total_size(size, downloaded=0): + global _text_meter_total_size + global _text_meter_sofar_size + _text_meter_total_size = size + _text_meter_sofar_size = downloaded + class TextMeter(BaseMeter): def __init__(self, fo=sys.stderr): BaseMeter.__init__(self) @@ -97,6 +108,12 @@ text = self.text else: text = self.basename + + sofar_size = None + if _text_meter_total_size: + sofar_size = _text_meter_sofar_size + amount_read + sofar_pc = (sofar_size * 100) / _text_meter_total_size + if self.size is None: out = '\r%-60.60s %5sB %s ' % \ (text, fread, fetime) @@ -104,15 +121,23 @@ rtime = self.re.remaining_time() frtime = format_time(rtime) frac = self.re.fraction_read() - bar = '='*int(25 * frac) - out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ETA ' % \ - (text, frac*100, bar, fread, frtime) + bar = '='*int(10 * frac) + ave_dl = format_number(self.re.average_rate()) + if sofar_size is None: + out = '\r%-25.25s %3i%% |%-14.14s| %5sB/s | %5sB %9s ETA ' % \ + (text, frac*100, bar, ave_dl, fread, frtime) + else: + fmt ='\r%-22.22s %3i%% |%4i%% |%-10.10s| %5sB/s | %5sB %9s ETA ' + out = fmt % (text, sofar_pc,frac*100,bar,ave_dl, fread,frtime) self.fo.write(out) self.fo.flush() def _do_end(self, amount_read, now=None): + global _text_meter_total_size + global _text_meter_sofar_size + total_time = format_time(self.re.elapsed_time()) total_size = format_number(amount_read) if self.text is not None: @@ -123,11 +148,16 @@ out = '\r%-60.60s %5sB %s ' % \ (text, total_size, total_time) else: - bar = '='*25 - out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ' % \ - (text, 100, bar, total_size, total_time) + out = '\r%-56.56s | %5sB %9s ' % \ + (text, total_size, total_time) + self.fo.write(out + '\n') self.fo.flush() + if _text_meter_total_size: + _text_meter_sofar_size += amount_read + if _text_meter_total_size <= _text_meter_sofar_size: + _text_meter_total_size = 0 + _text_meter_sofar_size = 0 text_progress_meter = TextMeter @@ -396,10 +426,12 @@ #print 'times', now, self.last_update_time time_diff = now - self.last_update_time read_diff = amount_read - self.last_amount_read - self.last_update_time = now + # First update, on reget is the file size + if self.last_amount_read: + self.last_update_time = now + self.ave_rate = self._temporal_rolling_ave(\ + time_diff, read_diff, self.ave_rate, self.timescale) self.last_amount_read = amount_read - self.ave_rate = self._temporal_rolling_ave(\ - time_diff, read_diff, self.ave_rate, self.timescale) #print 'results', time_diff, read_diff, self.ave_rate ##################################################################### @@ -528,3 +560,49 @@ format = '%.0f%s%s' return(format % (float(number or 0), space, symbols[depth])) + +def _tst(fn, cur, tot, beg, size, *args): + tm = TextMeter() + text = "(%d/%d): %s" % (cur, tot, fn) + tm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size, text=text) + num = beg + off = 0 + for (inc, delay) in args: + off += 1 + while num < ((size * off) / len(args)): + num += inc + tm.update(num) + time.sleep(delay) + tm.end(size) + +if __name__ == "__main__": + # (1/2): subversion-1.4.4-7.x86_64.rpm 2.4 MB / 85 kB/s 00:28 + # (2/2): mercurial-0.9.5-6.fc8.x86_64.rpm 924 kB / 106 kB/s 00:08 + if True: + text_meter_total_size(1000 + 10000 + 10000 + 1000000 + 1000000 + + 1000000 + 10000 + 10000 + 10000 + 1000000) + _tst("sm-1.0.0-1.fc8.i386.rpm", 1, 10, 0, 1000, + (10, 0.2), (10, 0.1), (100, 0.25)) + _tst("s-1.0.1-1.fc8.i386.rpm", 2, 10, 0, 10000, + (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25)) + _tst("m-1.0.1-2.fc8.i386.rpm", 3, 10, 5000, 10000, + (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25)) + _tst("large-file-name-Foo-11.8.7-4.5.6.1.fc8.x86_64.rpm", 4, 10, 0, 1000000, + (1000, 0.2), (1000, 0.1), (10000, 0.1)) + _tst("large-file-name-Foo2-11.8.7-4.5.6.2.fc8.x86_64.rpm", 5, 10, + 500001, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1)) + _tst("large-file-name-Foo3-11.8.7-4.5.6.3.fc8.x86_64.rpm", 6, 10, + 750002, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1)) + _tst("large-file-name-Foo4-10.8.7-4.5.6.1.fc8.x86_64.rpm", 7, 10, 0, 10000, + (100, 0.1)) + _tst("large-file-name-Foo5-10.8.7-4.5.6.2.fc8.x86_64.rpm", 8, 10, + 5001, 10000, (100, 0.1)) + _tst("large-file-name-Foo6-10.8.7-4.5.6.3.fc8.x86_64.rpm", 9, 10, + 7502, 10000, (100, 0.1)) + _tst("large-file-name-Foox-9.8.7-4.5.6.1.fc8.x86_64.rpm", 10, 10, + 0, 1000000, (10, 0.5), + (10000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), + (10000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), + (10000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), + (10000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), + (10000, 0.1), (10000, 0.25))