From 850f513283e6e75beb4dc25659a195efe5af0d45 Mon Sep 17 00:00:00 2001 From: James Antill Date: Mon, 7 Apr 2008 17:11:24 +0000 Subject: [PATCH] - Fix the ftp byterange port problem: - Resolves: bug#419241 - Fixup the progress UI: - add function for total progress - add total progress percentagee current download line - add rate to current download line - use dead space when finished downloading - don't confuse download rate on regets. --- python-urlgrabber.spec | 16 +++- urlgrabber-progress-ui.patch | 151 +++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 urlgrabber-progress-ui.patch diff --git a/python-urlgrabber.spec b/python-urlgrabber.spec index f9b8ae6..05cc656 100644 --- a/python-urlgrabber.spec +++ b/python-urlgrabber.spec @@ -3,11 +3,13 @@ Summary: A high-level cross-protocol url-grabber Name: python-urlgrabber Version: 3.0.0 -Release: 5%{?dist} +Release: 6%{?dist} Source0: urlgrabber-%{version}.tar.gz Patch0: urlgrabber-keepalive.patch Patch1: urlgrabber-string-type.patch Patch2: urlgrabber-3.0.0-cleanup.patch +Patch3: urlgrabber-ftp-port.patch +Patch4: urlgrabber-progress-ui.patch License: LGPLv2+ Group: Development/Libraries BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root @@ -26,6 +28,8 @@ authentication, proxies and more. %patch0 -p0 %patch1 -p1 %patch2 -p1 +%patch3 -p1 +%patch4 -p1 %build python setup.py build @@ -45,6 +49,16 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/urlgrabber %changelog +* Mon Apr 7 2008 James Antill 3.0.0-6 +- Fix the ftp byterange port problem: +- Resolves: bug#419241 +- Fixup the progress UI: +- add function for total progress +- add total progress percentagee current download line +- add rate to current download line +- use dead space when finished downloading +- don't confuse download rate on regets. + * Sat Mar 15 2008 Robert Scheck 3.0.0-5 - Make sure, that *.egg-info is catched up during build diff --git a/urlgrabber-progress-ui.patch b/urlgrabber-progress-ui.patch new file mode 100644 index 0000000..c841436 --- /dev/null +++ b/urlgrabber-progress-ui.patch @@ -0,0 +1,151 @@ +Index: progress.py +=================================================================== +RCS file: /home/groups/urlgrabber/cvs-root/urlgrabber/urlgrabber/progress.py,v +retrieving revision 1.7 +diff -u -r1.7 progress.py +--- progress.py 19 Aug 2005 21:59:07 -0000 1.7 ++++ progress.py 7 Apr 2008 16:33:30 -0000 +@@ -83,6 +83,17 @@ + def _do_end(self, amount_read, now=None): + pass + ++# This is kind of a hack, but progress is gotten from grabber which doesn't ++# know about the total size to download. So we do this so we can get the data ++# out of band here. This will be "fixed" one way or anther soon. ++_text_meter_total_size = 0 ++_text_meter_sofar_size = 0 ++def text_meter_total_size(size, downloaded=0): ++ global _text_meter_total_size ++ global _text_meter_sofar_size ++ _text_meter_total_size = size ++ _text_meter_sofar_size = downloaded ++ + class TextMeter(BaseMeter): + def __init__(self, fo=sys.stderr): + BaseMeter.__init__(self) +@@ -97,6 +108,12 @@ + text = self.text + else: + text = self.basename ++ ++ sofar_size = None ++ if _text_meter_total_size: ++ sofar_size = _text_meter_sofar_size + amount_read ++ sofar_pc = (sofar_size * 100) / _text_meter_total_size ++ + if self.size is None: + out = '\r%-60.60s %5sB %s ' % \ + (text, fread, fetime) +@@ -104,15 +121,23 @@ + rtime = self.re.remaining_time() + frtime = format_time(rtime) + frac = self.re.fraction_read() +- bar = '='*int(25 * frac) + +- out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ETA ' % \ +- (text, frac*100, bar, fread, frtime) ++ bar = '='*int(10 * frac) ++ ave_dl = format_number(self.re.average_rate()) ++ if sofar_size is None: ++ out = '\r%-25.25s %3i%% |%-14.14s| %5sB/s | %5sB %9s ETA ' % \ ++ (text, frac*100, bar, ave_dl, fread, frtime) ++ else: ++ fmt ='\r%-22.22s %3i%% |%4i%% |%-10.10s| %5sB/s | %5sB %9s ETA ' ++ out = fmt % (text, sofar_pc,frac*100,bar,ave_dl, fread,frtime) + + self.fo.write(out) + self.fo.flush() + + def _do_end(self, amount_read, now=None): ++ global _text_meter_total_size ++ global _text_meter_sofar_size ++ + total_time = format_time(self.re.elapsed_time()) + total_size = format_number(amount_read) + if self.text is not None: +@@ -123,11 +148,16 @@ + out = '\r%-60.60s %5sB %s ' % \ + (text, total_size, total_time) + else: +- bar = '='*25 +- out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ' % \ +- (text, 100, bar, total_size, total_time) ++ out = '\r%-56.56s | %5sB %9s ' % \ ++ (text, total_size, total_time) ++ + self.fo.write(out + '\n') + self.fo.flush() ++ if _text_meter_total_size: ++ _text_meter_sofar_size += amount_read ++ if _text_meter_total_size <= _text_meter_sofar_size: ++ _text_meter_total_size = 0 ++ _text_meter_sofar_size = 0 + + text_progress_meter = TextMeter + +@@ -396,10 +426,12 @@ + #print 'times', now, self.last_update_time + time_diff = now - self.last_update_time + read_diff = amount_read - self.last_amount_read +- self.last_update_time = now ++ # First update, on reget is the file size ++ if self.last_amount_read: ++ self.last_update_time = now ++ self.ave_rate = self._temporal_rolling_ave(\ ++ time_diff, read_diff, self.ave_rate, self.timescale) + self.last_amount_read = amount_read +- self.ave_rate = self._temporal_rolling_ave(\ +- time_diff, read_diff, self.ave_rate, self.timescale) + #print 'results', time_diff, read_diff, self.ave_rate + + ##################################################################### +@@ -528,3 +560,49 @@ + format = '%.0f%s%s' + + return(format % (float(number or 0), space, symbols[depth])) ++ ++def _tst(fn, cur, tot, beg, size, *args): ++ tm = TextMeter() ++ text = "(%d/%d): %s" % (cur, tot, fn) ++ tm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size, text=text) ++ num = beg ++ off = 0 ++ for (inc, delay) in args: ++ off += 1 ++ while num < ((size * off) / len(args)): ++ num += inc ++ tm.update(num) ++ time.sleep(delay) ++ tm.end(size) ++ ++if __name__ == "__main__": ++ # (1/2): subversion-1.4.4-7.x86_64.rpm 2.4 MB / 85 kB/s 00:28 ++ # (2/2): mercurial-0.9.5-6.fc8.x86_64.rpm 924 kB / 106 kB/s 00:08 ++ if True: ++ text_meter_total_size(1000 + 10000 + 10000 + 1000000 + 1000000 + ++ 1000000 + 10000 + 10000 + 10000 + 1000000) ++ _tst("sm-1.0.0-1.fc8.i386.rpm", 1, 10, 0, 1000, ++ (10, 0.2), (10, 0.1), (100, 0.25)) ++ _tst("s-1.0.1-1.fc8.i386.rpm", 2, 10, 0, 10000, ++ (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25)) ++ _tst("m-1.0.1-2.fc8.i386.rpm", 3, 10, 5000, 10000, ++ (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25)) ++ _tst("large-file-name-Foo-11.8.7-4.5.6.1.fc8.x86_64.rpm", 4, 10, 0, 1000000, ++ (1000, 0.2), (1000, 0.1), (10000, 0.1)) ++ _tst("large-file-name-Foo2-11.8.7-4.5.6.2.fc8.x86_64.rpm", 5, 10, ++ 500001, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1)) ++ _tst("large-file-name-Foo3-11.8.7-4.5.6.3.fc8.x86_64.rpm", 6, 10, ++ 750002, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1)) ++ _tst("large-file-name-Foo4-10.8.7-4.5.6.1.fc8.x86_64.rpm", 7, 10, 0, 10000, ++ (100, 0.1)) ++ _tst("large-file-name-Foo5-10.8.7-4.5.6.2.fc8.x86_64.rpm", 8, 10, ++ 5001, 10000, (100, 0.1)) ++ _tst("large-file-name-Foo6-10.8.7-4.5.6.3.fc8.x86_64.rpm", 9, 10, ++ 7502, 10000, (100, 0.1)) ++ _tst("large-file-name-Foox-9.8.7-4.5.6.1.fc8.x86_64.rpm", 10, 10, ++ 0, 1000000, (10, 0.5), ++ (10000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), ++ (10000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), ++ (10000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), ++ (10000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1), ++ (10000, 0.1), (10000, 0.25))