python-urlgrabber/urlgrabber-progress-ui.patch

diff -rup urlgrabber-3.0.0-orig/urlgrabber/progress.py urlgrabber-3.0.0/urlgrabber/progress.py
--- urlgrabber-3.0.0-orig/urlgrabber/progress.py	2008-06-16 00:48:52.000000000 -0400
+++ urlgrabber-3.0.0/urlgrabber/progress.py	2008-06-16 00:49:25.000000000 -0400
@@ -24,7 +24,74 @@ import time
 import math
 import thread
 import types
-    
+import fcntl
+import struct
+import termios
+
+# Code from http://mail.python.org/pipermail/python-list/2000-May/033365.html
+def terminal_width(fd=1):
+    """ Get the real terminal width """
+    try:
+        buf = 'abcdefgh'
+        buf = fcntl.ioctl(fd, termios.TIOCGWINSZ, buf)
+        return struct.unpack('hhhh', buf)[1]
+    except: # IOError
+        return 80
+
+_term_width_val  = None
+_term_width_last = None
+def terminal_width_cached(fd=1, cache_timeout=1.000):
+    """ Get the real terminal width, but cache it for a bit. """
+    global _term_width_val
+    global _term_width_last
+
+    now = time.time()
+    if _term_width_val is None or (now - _term_width_last) > cache_timeout:
+        _term_width_val  = terminal_width(fd)
+        _term_width_last = now
+    return _term_width_val
+
+class TerminalLine:
+    """ Help create dynamic progress bars, uses terminal_width_cached(). """
+
+    def __init__(self, min_rest=0, beg_len=None, fd=1, cache_timeout=1.000):
+        if beg_len is None:
+            beg_len = min_rest
+        self._min_len = min_rest
+        self._llen    = terminal_width_cached(fd, cache_timeout)
+        if self._llen < beg_len:
+            self._llen = beg_len
+        self._fin = False
+
+    def __len__(self):
+        """ Usable length for elements. """
+        return self._llen - self._min_len
+
+    def rest_split(self, fixed, elements=2):
+        """ After a fixed length, split the rest of the line length among
+            a number of different elements (default=2). """
+        if self._llen < fixed:
+            return 0
+        return (self._llen - fixed) / elements
+
+    def add(self, element, full_len=None):
+        """ If there is room left in the line, above min_len, add element.
+            Note that as soon as one add fails all the rest will fail too. """
+
+        if full_len is None:
+            full_len = len(element)
+        if len(self) < full_len:
+            self._fin = True
+        if self._fin:
+            return ''
+
+        self._llen -= len(element)
+        return element
+
+    def rest(self):
+        """ Current rest of line, same as .rest_split(fixed=0, elements=1). """
+        return self._llen
+
 class BaseMeter:
     def __init__(self):
         self.update_period = 0.3 # seconds
@@ -84,6 +151,64 @@ class BaseMeter:
     def _do_end(self, amount_read, now=None):
         pass
         
+#  This is kind of a hack, but progress is gotten from grabber which doesn't
+# know about the total size to download. So we do this so we can get the data
+# out of band here. This will be "fixed" one way or anther soon.
+_text_meter_total_size = 0
+_text_meter_sofar_size = 0
+def text_meter_total_size(size, downloaded=0):
+    global _text_meter_total_size
+    global _text_meter_sofar_size
+    _text_meter_total_size = size
+    _text_meter_sofar_size = downloaded
+
+#
+#       update: No size (minimal: 17 chars)
+#       -----------------------------------
+# <text>                          <rate> | <current size> <elapsed time> 
+#  8-48                          1    8  3             6 1            9 5
+#
+# Order: 1. <text>+<current size> (17)
+#        2. +<elapsed time>       (10, total: 27)
+#        3. +                     ( 5, total: 32)
+#        4. +<rate>               ( 9, total: 41)
+#
+#       update: Size, Single file
+#       -------------------------
+# <text>            <pc>  <bar> <rate> | <current size> <eta time> ETA
+#  8-25            1 3-4 1 6-16 1   8  3             6 1        9 1  3 1
+#
+# Order: 1. <text>+<current size> (17)
+#        2. +<eta time>           (10, total: 27)
+#        3. +ETA                  ( 5, total: 32)
+#        4. +<pc>                 ( 4, total: 36)
+#        5. +<rate>               ( 9, total: 45)
+#        6. +<bar>                ( 7, total: 52)
+#
+#       update: Size, All files
+#       -----------------------
+# <text> <total pc> <pc>  <bar> <rate> | <current size> <eta time> ETA
+#  8-22 1      5-7 1 3-4 1 6-12 1   8  3             6 1        9 1  3 1
+#
+# Order: 1. <text>+<current size> (17)
+#        2. +<eta time>           (10, total: 27)
+#        3. +ETA                  ( 5, total: 32)
+#        4. +<total pc>           ( 5, total: 37)
+#        4. +<pc>                 ( 4, total: 41)
+#        5. +<rate>               ( 9, total: 50)
+#        6. +<bar>                ( 7, total: 57)
+#
+#       end
+#       ---
+# <text>                                 | <current size> <elapsed time> 
+#  8-56                                  3             6 1            9 5
+#
+# Order: 1. <text>                ( 8)
+#        2. +<current size>       ( 9, total: 17)
+#        3. +<elapsed time>       (10, total: 27)
+#        4. +                     ( 5, total: 32)
+#
+
 class TextMeter(BaseMeter):
     def __init__(self, fo=sys.stderr):
         BaseMeter.__init__(self)
@@ -98,38 +223,80 @@ class TextMeter(BaseMeter):
             text = self.text
         else:
             text = self.basename
+
+        ave_dl = format_number(self.re.average_rate())
+        sofar_size = None
+        if _text_meter_total_size:
+            sofar_size = _text_meter_sofar_size + amount_read
+            sofar_pc   = (sofar_size * 100) / _text_meter_total_size
+
+        # Include text + ui_rate in minimal
+        tl = TerminalLine(8, 8+1+8)
+        ui_size = tl.add(' | %5sB' % fread)
         if self.size is None:
-            out = '\r%-60.60s    %5sB %s ' % \
-                  (text, fread, fetime)
+            ui_time = tl.add(' %9s' % fetime)
+            ui_end  = tl.add(' ' * 5)
+            ui_rate = tl.add(' %5sB/s' % ave_dl)
+            out = '\r%-*.*s%s%s%s%s' % (tl.rest(), tl.rest(), text,
+                                        ui_rate, ui_size, ui_time, ui_end)
         else:
             rtime = self.re.remaining_time()
             frtime = format_time(rtime)
             frac = self.re.fraction_read()
-            bar = '='*int(25 * frac)
 
-            out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ETA ' % \
-                  (text, frac*100, bar, fread, frtime)
+            ui_time = tl.add(' %9s' % frtime)
+            ui_end  = tl.add(' ETA ')
+
+            if sofar_size is None:
+                ui_sofar_pc = ''
+            else:
+                ui_sofar_pc = tl.add(' (%i%%)' % sofar_pc,
+                                     full_len=len(" (100%)"))
+
+            ui_pc   = tl.add(' %2i%%' % (frac*100))
+            ui_rate = tl.add(' %5sB/s' % ave_dl)
+            # Make text grow a bit before we start growing the bar too
+            blen = 4 + tl.rest_split(8 + 8 + 4)
+            bar  = '='*int(blen * frac)
+            ui_bar  = tl.add(' [%-*.*s]' % (blen, blen, bar))
+            out = '\r%-*.*s%s%s%s%s%s%s%s' % (tl.rest(), tl.rest(), text,
+                                              ui_sofar_pc, ui_pc, ui_bar,
+                                              ui_rate, ui_size, ui_time, ui_end)
 
         self.fo.write(out)
         self.fo.flush()
 
     def _do_end(self, amount_read, now=None):
+        global _text_meter_total_size
+        global _text_meter_sofar_size
+
         total_time = format_time(self.re.elapsed_time())
         total_size = format_number(amount_read)
         if self.text is not None:
             text = self.text
         else:
             text = self.basename
-        if self.size is None:
-            out = '\r%-60.60s    %5sB %s ' % \
-                  (text, total_size, total_time)
-        else:
-            bar = '='*25
-            out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s     ' % \
-                  (text, 100, bar, total_size, total_time)
-        self.fo.write(out + '\n')
+
+        tl = TerminalLine(8)
+        ui_size = tl.add(' | %5sB' % total_size)
+        ui_time = tl.add(' %9s' % total_time)
+        ui_end  = tl.add(' ' * 5)
+        out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text,
+                                    ui_size, ui_time, ui_end)
+        self.fo.write(out)
         self.fo.flush()
 
+        # Don't add size to the sofar size until we have all of it.
+        # If we don't have a size, then just pretend/hope we got all of it.
+        if self.size is not None and amount_read != self.size:
+            return
+
+        if _text_meter_total_size:
+            _text_meter_sofar_size += amount_read
+        if _text_meter_total_size <= _text_meter_sofar_size:
+            _text_meter_total_size = 0
+            _text_meter_sofar_size = 0
+
 text_progress_meter = TextMeter
 
 class MultiFileHelper(BaseMeter):
@@ -397,10 +564,12 @@ class RateEstimator:
         #print 'times', now, self.last_update_time
         time_diff = now         - self.last_update_time
         read_diff = amount_read - self.last_amount_read
-        self.last_update_time = now
+        # First update, on reget is the file size
+        if self.last_amount_read:
+            self.last_update_time = now
+            self.ave_rate = self._temporal_rolling_ave(\
+                time_diff, read_diff, self.ave_rate, self.timescale)
         self.last_amount_read = amount_read
-        self.ave_rate = self._temporal_rolling_ave(\
-            time_diff, read_diff, self.ave_rate, self.timescale)
         #print 'results', time_diff, read_diff, self.ave_rate
         
     #####################################################################
@@ -529,3 +698,49 @@ def format_number(number, SI=0, space=' 
         format = '%.0f%s%s'
         
     return(format % (float(number or 0), space, symbols[depth]))
+
+def _tst(fn, cur, tot, beg, size, *args):
+    tm = TextMeter()
+    text = "(%d/%d): %s" % (cur, tot, fn)
+    tm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size, text=text)
+    num = beg
+    off = 0
+    for (inc, delay) in args:
+        off += 1
+        while num < ((size * off) / len(args)):
+            num += inc
+            tm.update(num)
+            time.sleep(delay)
+    tm.end(size)
+
+if __name__ == "__main__":
+    # (1/2): subversion-1.4.4-7.x86_64.rpm               2.4 MB /  85 kB/s    00:28     
+    # (2/2): mercurial-0.9.5-6.fc8.x86_64.rpm            924 kB / 106 kB/s    00:08     
+    if len(sys.argv) >= 2 and sys.argv[1] == 'total':
+        text_meter_total_size(1000 + 10000 + 10000 + 1000000 + 1000000 +
+                              1000000 + 10000 + 10000 + 10000 + 1000000)
+    _tst("sm-1.0.0-1.fc8.i386.rpm", 1, 10, 0, 1000,
+         (10, 0.2), (10, 0.1), (100, 0.25))
+    _tst("s-1.0.1-1.fc8.i386.rpm", 2, 10, 0, 10000,
+         (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
+    _tst("m-1.0.1-2.fc8.i386.rpm", 3, 10, 5000, 10000,
+         (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
+    _tst("large-file-name-Foo-11.8.7-4.5.6.1.fc8.x86_64.rpm", 4, 10, 0, 1000000,
+         (1000, 0.2), (1000, 0.1), (10000, 0.1))
+    _tst("large-file-name-Foo2-11.8.7-4.5.6.2.fc8.x86_64.rpm", 5, 10,
+         500001, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1))
+    _tst("large-file-name-Foo3-11.8.7-4.5.6.3.fc8.x86_64.rpm", 6, 10,
+         750002, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1))
+    _tst("large-file-name-Foo4-10.8.7-4.5.6.1.fc8.x86_64.rpm", 7, 10, 0, 10000,
+         (100, 0.1))
+    _tst("large-file-name-Foo5-10.8.7-4.5.6.2.fc8.x86_64.rpm", 8, 10,
+         5001, 10000, (100, 0.1))
+    _tst("large-file-name-Foo6-10.8.7-4.5.6.3.fc8.x86_64.rpm", 9, 10,
+         7502, 10000, (1, 0.1))
+    _tst("large-file-name-Foox-9.8.7-4.5.6.1.fc8.x86_64.rpm",  10, 10,
+         0, 1000000, (10, 0.5),
+         (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
+         (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
+         (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
+         (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
+         (100000, 0.1), (1, 0.1))
Only in urlgrabber-3.0.0/urlgrabber: progress.py.orig