Index: urlgrabber/progress.py =================================================================== RCS file: /home/groups/urlgrabber/cvs-root/urlgrabber/urlgrabber/progress.py,v retrieving revision 1.7 diff -u -r1.7 progress.py --- urlgrabber/progress.py 19 Aug 2005 21:59:07 -0000 1.7 +++ urlgrabber/progress.py 17 May 2008 17:33:32 -0000 @@ -23,8 +23,75 @@ import time import math import thread import types - +import fcntl +import struct +import termios + +# Code from http://mail.python.org/pipermail/python-list/2000-May/033365.html +def terminal_width(fd=1): + """ Get the real terminal width """ + try: + buf = 'abcdefgh' + buf = fcntl.ioctl(fd, termios.TIOCGWINSZ, buf) + return struct.unpack('hhhh', buf)[1] + except: # IOError + return 80 + +_term_width_val = None +_term_width_last = None +def terminal_width_cached(fd=1, cache_timeout=1.000): + """ Get the real terminal width, but cache it for a bit. """ + global _term_width_val + global _term_width_last + + now = time.time() + if _term_width_val is None or (now - _term_width_last) > cache_timeout: + _term_width_val = terminal_width(fd) + _term_width_last = now + return _term_width_val + +class TerminalLine: + """ Help create dynamic progress bars, uses terminal_width_cached(). """ + + def __init__(self, min_rest=0, beg_len=None, fd=1, cache_timeout=1.000): + if beg_len is None: + beg_len = min_rest + self._min_len = min_rest + self._llen = terminal_width_cached(fd, cache_timeout) + if self._llen < beg_len: + self._llen = beg_len + self._fin = False + + def __len__(self): + """ Usable length for elements. """ + return self._llen - self._min_len + + def rest_split(self, fixed, elements=2): + """ After a fixed length, split the rest of the line length among + a number of different elements (default=2). """ + if self._llen < fixed: + return 0 + return (self._llen - fixed) / elements + + def add(self, element, full_len=None): + """ If there is room left in the line, above min_len, add element. + Note that as soon as one add fails all the rest will fail too. """ + + if full_len is None: + full_len = len(element) + if len(self) < full_len: + self._fin = True + if self._fin: + return '' + + self._llen -= len(element) + return element + + def rest(self): + """ Current rest of line, same as .rest_split(fixed=0, elements=1). """ + return self._llen + class BaseMeter: def __init__(self): self.update_period = 0.3 # seconds @@ -83,6 +150,64 @@ def _do_end(self, amount_read, now=None): pass +# This is kind of a hack, but progress is gotten from grabber which doesn't +# know about the total size to download. So we do this so we can get the data +# out of band here. This will be "fixed" one way or anther soon. +_text_meter_total_size = 0 +_text_meter_sofar_size = 0 +def text_meter_total_size(size, downloaded=0): + global _text_meter_total_size + global _text_meter_sofar_size + _text_meter_total_size = size + _text_meter_sofar_size = downloaded + +# +# update: No size (minimal: 17 chars) +# ----------------------------------- +# | +# 8-48 1 8 3 6 1 9 5 +# +# Order: 1. + (17) +# 2. + (10, total: 27) +# 3. + ( 5, total: 32) +# 4. + ( 9, total: 41) +# +# update: Size, Single file +# ------------------------- +# | ETA +# 8-25 1 3-4 1 6-16 1 8 3 6 1 9 1 3 1 +# +# Order: 1. + (17) +# 2. + (10, total: 27) +# 3. +ETA ( 5, total: 32) +# 4. + ( 4, total: 36) +# 5. + ( 9, total: 45) +# 6. + ( 7, total: 52) +# +# update: Size, All files +# ----------------------- +# | ETA +# 8-22 1 5-7 1 3-4 1 6-12 1 8 3 6 1 9 1 3 1 +# +# Order: 1. + (17) +# 2. + (10, total: 27) +# 3. +ETA ( 5, total: 32) +# 4. + ( 5, total: 37) +# 4. + ( 4, total: 41) +# 5. + ( 9, total: 50) +# 6. + ( 7, total: 57) +# +# end +# --- +# | +# 8-56 3 6 1 9 5 +# +# Order: 1. ( 8) +# 2. + ( 9, total: 17) +# 3. + (10, total: 27) +# 4. + ( 5, total: 32) +# + class TextMeter(BaseMeter): def __init__(self, fo=sys.stderr): BaseMeter.__init__(self) @@ -97,37 +222,73 @@ text = self.text else: text = self.basename + + ave_dl = format_number(self.re.average_rate()) + sofar_size = None + if _text_meter_total_size: + sofar_size = _text_meter_sofar_size + amount_read + sofar_pc = (sofar_size * 100) / _text_meter_total_size + + # Include text + ui_rate in minimal + tl = TerminalLine(8, 8+1+8) + ui_size = tl.add(' | %5sB' % fread) if self.size is None: - out = '\r%-60.60s %5sB %s ' % \ - (text, fread, fetime) + ui_time = tl.add(' %9s' % fetime) + ui_end = tl.add(' ' * 5) + ui_rate = tl.add(' %5sB/s' % ave_dl) + out = '\r%-*.*s%s%s%s%s' % (tl.rest(), tl.rest(), text, + ui_rate, ui_size, ui_time, ui_end) else: rtime = self.re.remaining_time() frtime = format_time(rtime) frac = self.re.fraction_read() - bar = '='*int(25 * frac) - out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ETA ' % \ - (text, frac*100, bar, fread, frtime) + ui_time = tl.add(' %9s' % frtime) + ui_end = tl.add(' ETA ') + + if sofar_size is None: + ui_sofar_pc = '' + else: + ui_sofar_pc = tl.add(' (%i%%)' % sofar_pc, + full_len=len(" (100%)")) + + ui_pc = tl.add(' %2i%%' % (frac*100)) + ui_rate = tl.add(' %5sB/s' % ave_dl) + # Make text grow a bit before we start growing the bar too + blen = 4 + tl.rest_split(8 + 8 + 4) + bar = '='*int(blen * frac) + ui_bar = tl.add(' [%-*.*s]' % (blen, blen, bar)) + out = '\r%-*.*s%s%s%s%s%s%s%s' % (tl.rest(), tl.rest(), text, + ui_sofar_pc, ui_pc, ui_bar, + ui_rate, ui_size, ui_time, ui_end) self.fo.write(out) self.fo.flush() def _do_end(self, amount_read, now=None): + global _text_meter_total_size + global _text_meter_sofar_size + total_time = format_time(self.re.elapsed_time()) total_size = format_number(amount_read) if self.text is not None: text = self.text else: text = self.basename - if self.size is None: - out = '\r%-60.60s %5sB %s ' % \ - (text, total_size, total_time) - else: - bar = '='*25 - out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ' % \ - (text, 100, bar, total_size, total_time) - self.fo.write(out + '\n') + + tl = TerminalLine(8) + ui_size = tl.add(' | %5sB' % total_size) + ui_time = tl.add(' %9s' % total_time) + ui_end = tl.add(' ' * 5) + out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text, + ui_size, ui_time, ui_end) + self.fo.write(out) self.fo.flush() + if _text_meter_total_size: + _text_meter_sofar_size += amount_read + if _text_meter_total_size <= _text_meter_sofar_size: + _text_meter_total_size = 0 + _text_meter_sofar_size = 0 text_progress_meter = TextMeter @@ -396,10 +557,12 @@ #print 'times', now, self.last_update_time time_diff = now - self.last_update_time read_diff = amount_read - self.last_amount_read - self.last_update_time = now + # First update, on reget is the file size + if self.last_amount_read: + self.last_update_time = now + self.ave_rate = self._temporal_rolling_ave(\ + time_diff, read_diff, self.ave_rate, self.timescale) self.last_amount_read = amount_read - self.ave_rate = self._temporal_rolling_ave(\ - time_diff, read_diff, self.ave_rate, self.timescale) #print 'results', time_diff, read_diff, self.ave_rate ##################################################################### @@ -528,3 +691,49 @@ format = '%.0f%s%s' return(format % (float(number or 0), space, symbols[depth])) + +def _tst(fn, cur, tot, beg, size, *args): + tm = TextMeter() + text = "(%d/%d): %s" % (cur, tot, fn) + tm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size, text=text) + num = beg + off = 0 + for (inc, delay) in args: + off += 1 + while num < ((size * off) / len(args)): + num += inc + tm.update(num) + time.sleep(delay) + tm.end(size) + +if __name__ == "__main__": + # (1/2): subversion-1.4.4-7.x86_64.rpm 2.4 MB / 85 kB/s 00:28 + # (2/2): mercurial-0.9.5-6.fc8.x86_64.rpm 924 kB / 106 kB/s 00:08 + if len(sys.argv) >= 2 and sys.argv[1] == 'total': + text_meter_total_size(1000 + 10000 + 10000 + 1000000 + 1000000 + + 1000000 + 10000 + 10000 + 10000 + 1000000) + _tst("sm-1.0.0-1.fc8.i386.rpm", 1, 10, 0, 1000, + (10, 0.2), (10, 0.1), (100, 0.25)) + _tst("s-1.0.1-1.fc8.i386.rpm", 2, 10, 0, 10000, + (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25)) + _tst("m-1.0.1-2.fc8.i386.rpm", 3, 10, 5000, 10000, + (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25)) + _tst("large-file-name-Foo-11.8.7-4.5.6.1.fc8.x86_64.rpm", 4, 10, 0, 1000000, + (1000, 0.2), (1000, 0.1), (10000, 0.1)) + _tst("large-file-name-Foo2-11.8.7-4.5.6.2.fc8.x86_64.rpm", 5, 10, + 500001, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1)) + _tst("large-file-name-Foo3-11.8.7-4.5.6.3.fc8.x86_64.rpm", 6, 10, + 750002, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1)) + _tst("large-file-name-Foo4-10.8.7-4.5.6.1.fc8.x86_64.rpm", 7, 10, 0, 10000, + (100, 0.1)) + _tst("large-file-name-Foo5-10.8.7-4.5.6.2.fc8.x86_64.rpm", 8, 10, + 5001, 10000, (100, 0.1)) + _tst("large-file-name-Foo6-10.8.7-4.5.6.3.fc8.x86_64.rpm", 9, 10, + 7502, 10000, (1, 0.1)) + _tst("large-file-name-Foox-9.8.7-4.5.6.1.fc8.x86_64.rpm", 10, 10, + 0, 1000000, (10, 0.5), + (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), + (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), + (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), + (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), + (100000, 0.1), (1, 0.1))