tags 488415 + patch
thanks
The attached patch adds the total amount downloaded via HTTP to
the status page like this:
---------------------------------
... s|download/|downloaded (HTTP)/| ...
| upload| uploaded|
-+---------+------------------+--
... 4| 0.0K/s| 15.0MiB (317KiB)|12 ...
| 0.0K/s| 4.24MiB|
-+---------+------------------+--
... 1| 0.0K/s| 104MiB (0B)|1. ...
| 0.0K/s| 9.63MiB|
---------------------------------
HTTP downloading increases both the main "downloaded" stat and
the one in brackets.
Downgrading back to debtorrent-0.1.8 works fine. It will lose
the HTTP count, but leave everything else OK. I haven't changed
the version number of the state file to reflect the extra data.
This patch and the one for #487829 should be able to go in either
order; but for my development I've done this on top of #487829.
Steve
Mon Jun 30 00:28:28 BST 2008 Steve Cotton <[EMAIL PROTECTED]>
* #488415 Unnecessary but tidy correction to handling state files from 0.1.8
Mon Jun 30 00:09:54 BST 2008 Steve Cotton <[EMAIL PROTECTED]>
* #488415 tiny bit of spelling and grammar
Sun Jun 29 23:56:33 BST 2008 Steve Cotton <[EMAIL PROTECTED]>
* #488415 Call a subtotal a subtotal
Sun Jun 29 23:30:35 BST 2008 Steve Cotton <[EMAIL PROTECTED]>
* #488415 Show the amount downloaded via HTTP
Sun Jun 29 18:26:13 BST 2008 Steve Cotton <[EMAIL PROTECTED]>
* #488415 debugging: remove debug code (still needs the implementation to be tidied)
Sun Jun 29 18:13:42 BST 2008 Steve Cotton <[EMAIL PROTECTED]>
* #488415 debugging: split the download total in to (downloaded by torrent) and (downloaded by HTTP)
Partial change only - this has the initial (functional) prototype code,
along with a lot of debugging. Need to remove the debugging.
diff -rN -u old-debtorrent/debian/changelog new-debtorrent/debian/changelog
--- old-debtorrent/debian/changelog 2008-06-30 00:31:03.000000000 +0100
+++ new-debtorrent/debian/changelog 2008-06-30 00:31:03.000000000 +0100
@@ -1,3 +1,9 @@
+debtorrent (0.1.8.steve2.2) norelease; urgency=low
+
+ * Show the amount of data downloaded via HTTP (Closes: #488415)
+
+ -- Steve Cotton <[EMAIL PROTECTED]> Sun, 29 Jun 2008 22:08:52 +0100
+
debtorrent (0.1.8.steve2.1) norelease; urgency=low
* Fix: download / upload stats are correct after
diff -rN -u old-debtorrent/DebTorrent/BT1/AptListener.py new-debtorrent/DebTorrent/BT1/AptListener.py
--- old-debtorrent/DebTorrent/BT1/AptListener.py 2008-06-30 00:31:03.000000000 +0100
+++ new-debtorrent/DebTorrent/BT1/AptListener.py 2008-06-30 00:31:03.000000000 +0100
@@ -275,7 +275,7 @@
'<th align="right">distributed copies</th>\n' \
'<th align="right">download/<br>\n' \
'upload</th>\n' \
- '<th align="right">downloaded/<br>\n' \
+ '<th align="right">downloaded (HTTP)/<br>\n' \
'uploaded</th>\n' \
'<th align="right">size</th>\n' \
'<th align="right">time remaining</th>\n' \
@@ -289,13 +289,10 @@
# Display a table row for each running torrent
for x in data:
- ( name, hash, status, progress, peers, seeds, seedsmsg, dist,
- uprate, dnrate, upamt, dnamt, size, t, msg ) = x
-
if self.allow_get:
- linkname = '<a href="/file?info_hash=' + quote(hash) + '">' + name + '</a>'
+ linkname = '<a href="/file?info_hash=' + quote(x.id) + '">' + x.name + '</a>'
else:
- linkname = name
+ linkname = x.name
s.write('<tr>\n'
'<td>%s<br>\n' \
@@ -307,14 +304,14 @@
'<td align="right">%.3f</td>\n' \
'<td align="right">%0.1fK/s<br>\n' \
'%0.1fK/s</td>\n' \
- '<td align="right">%s<br>\n' \
+ '<td align="right">%s (%s)<br>\n' \
'%s</td>\n' \
'<td align="right">%s</td>\n' \
'<td align="right">%s</td>\n' \
'<td>%s</td></tr>\n' \
- % (linkname, b2a_hex(hash), status, progress, peers, seeds,
- dist, dnrate/1000, uprate/1000, size_format(dnamt),
- size_format(upamt), size_format(size), hours(t), msg))
+ % (linkname, b2a_hex(x.id), x.status, x.progress, x.numPeers, x.numSeeds,
+ x.numCopies, x.downRate/1000, x.upRate/1000, size_format(x.downTotal), size_format(x.httpDownTotal),
+ size_format(x.upTotal), size_format(x.size), hours(x.timeRemaining), x.errorMessage))
s.write('</table>\n' \
'<ul>\n' \
@@ -559,11 +556,8 @@
total_dnrate = 0
for x in data:
- ( name, hash, status, progress, peers, seeds, seedsmsg, dist,
- uprate, dnrate, upamt, dnamt, size, t, msg ) = x
-
- total_size += size
- total_dnrate += dnrate
+ total_size += x.size
+ total_dnrate += x.downRate
message += ' at ' + size_format(total_dnrate) + '/s'
if total_dnrate > 0:
diff -rN -u old-debtorrent/DebTorrent/BT1/Statistics.py new-debtorrent/DebTorrent/BT1/Statistics.py
--- old-debtorrent/DebTorrent/BT1/Statistics.py 2008-06-30 00:31:03.000000000 +0100
+++ new-debtorrent/DebTorrent/BT1/Statistics.py 2008-06-30 00:31:03.000000000 +0100
@@ -1,5 +1,6 @@
# Written by Edward Keyes
# Modified by Cameron Dale
+# Modified by Steve Cotton
# see LICENSE.txt for license information
#
# $Id: Statistics.py 266 2007-08-18 02:06:35Z camrdale-guest $
@@ -12,6 +13,71 @@
"""Empty class to add arbitrary variables to."""
pass
+class CookedStatistics:
+ """Information returned from LaunchMany.gather_statistics.
+
+ These map to the similarly-named stats on the local HTTP
+ status page.
+
+ The class is named "cooked" because much of this comes from
+ the raw data in C{Statistics_Response}.
+
+ @type name: C{string}
+ @ivar name: The torrent name (dists_stable_main_binary-all etc.)
+ @type id: C{string}
+ @ivar id: Torrent identifier (the long-lived one).
+ @type status: C{string}
+ @ivar status: Local status message.
+ @type progress: C{string}
+ @ivar progress: This is a percentage, expressed as a string.
+ @type numPeers: C{int}
+ @ivar numPeers: Number of peers (excluding seeds).
+ @type numSeeds: C{int}
+ @ivar numSeeds: Number of seeds.
+ @type numCopies: C{float}
+ @ivar numCopies: Number of distributed copies.
+ @type upRate: C{float}
+ @ivar upRate: The rate that data is being uploaded.
+ @type downRate: C{float}
+ @ivar downRate: The rate that data is being downloaded, via any method.
+ @type upTotal: C{long}
+ @ivar upTotal: The amount uploaded.
+ @type downTotal: C{long}
+ @ivar downTotal: The amount downloaded, via any method.
+ @type downHttpTotal: C{long}
+ @ivar downHttpTotal: The amount downloaded via HTTP, this is also included in the downTotal.
+ @type size: C{long}
+ @ivar size: The size of the files already on disk, plus the amount queued for download.
+ @type timeRemaining: C{float}
+ @ivar timeRemaining: Estimated time for the download to complete (in hours).
+ @type errorMessage: C{string}
+ @ivar errorMessage: Last error message, if any.
+
+ """
+
+ def __init__(self, name, id):
+ """Initialize.
+
+ The torrent name and id always need to be supplied,
+ everything else can have sensible defaults.
+
+ """
+
+ self.name = name
+ self.id = id
+ self.status = ""
+ self.progress = "0.0%"
+ self.numPeers = 0
+ self.numSeeds = 0
+ self.numCopies = 0
+ self.upRate = 0.0
+ self.downRate = 0.0
+ self.upTotal = 0L
+ self.downTotal = 0L
+ self.httpDownTotal = 0L
+ self.size = 0L
+ self.timeRemaining = 0.0
+ self.errorMessage = ""
class Statistics:
"""Generate statistics for the swarm.
@@ -162,6 +228,7 @@
s = Statistics_Response()
s.upTotal = self.upmeasure.get_total()
s.downTotal = self.downmeasure.get_total()
+ s.httpDownTotal = self.downmeasure.get_http_subtotal()
s.last_failed = self.rerequest_lastfailed()
s.external_connection_made = self.connecter.external_connection_made
if s.downTotal > 0:
diff -rN -u old-debtorrent/DebTorrent/CurrentRateMeasure.py new-debtorrent/DebTorrent/CurrentRateMeasure.py
--- old-debtorrent/DebTorrent/CurrentRateMeasure.py 2008-06-30 00:31:03.000000000 +0100
+++ new-debtorrent/DebTorrent/CurrentRateMeasure.py 2008-06-30 00:31:03.000000000 +0100
@@ -10,7 +10,14 @@
class Measure:
"""The measurement of one rate.
-
+
+ This keeps track of both the current rate and the total
+ amount of data sent or received.
+
+ For DebTorrent, it can keep a subtotal of the amount of data
+ transferred via HTTP. Currently the HTTP rate is not
+ calculated, just the total.
+
@type max_rate_period: C{float}
@ivar max_rate_period: maximum amount of time to guess the current rate
estimate represents
@@ -22,10 +29,12 @@
@ivar rate: the latest calculated rate
@type total: C{long}
@ivar total: the total amount that went in to calculating the rate
+ @type httptotal: C{long}
+ @ivar httptotal: the total amount of HTTP data (also included in the total)
"""
- def __init__(self, max_rate_period, fudge = 1, saved_total = 0L):
+ def __init__(self, max_rate_period, fudge = 1, saved_total = 0L, http_total = 0L):
"""Initialize the measurement.
@type max_rate_period: C{float}
@@ -37,6 +46,8 @@
@type saved_total: C{long}
@param saved_total: the saved amount measured from a previous run
(optional, defaults to 0)
+ @param http_total: the saved amount measured from a previous run
+ (optional, defaults to 0)
"""
@@ -45,6 +56,7 @@
self.last = self.ratesince
self.rate = 0.0
self.total = long(saved_total)
+ self.httptotal = long(http_total)
def update_rate(self, amount):
"""Update the rate with new data.
@@ -62,6 +74,20 @@
if self.ratesince < t - self.max_rate_period:
self.ratesince = t - self.max_rate_period
+ def update_http_rate(self, amount):
+ """Update the rate with new data.
+
+ This new data is added to the figures returned by all of
+ C{get_rate}, C{get_total} and C{get_http_subtotal}.
+
+ @type amount: C{long}
+ @param amount: the new data to add into the rate calculation
+
+ """
+
+ self.httptotal += amount
+ self.update_rate(amount)
+
def get_rate(self):
"""Get the current rate measurement.
@@ -107,4 +133,14 @@
"""
- return self.total
\ No newline at end of file
+ return self.total
+
+ def get_http_subtotal(self):
+ """Get the amount transferred via HTTP only
+
+ @rtype: C{float}
+ @return: the total amount
+
+ """
+
+ return self.httptotal
diff -rN -u old-debtorrent/DebTorrent/download_bt1.py new-debtorrent/DebTorrent/download_bt1.py
--- old-debtorrent/DebTorrent/download_bt1.py 2008-06-30 00:31:03.000000000 +0100
+++ new-debtorrent/DebTorrent/download_bt1.py 2008-06-30 00:31:03.000000000 +0100
@@ -905,7 +905,7 @@
else:
self.pickled_data['resume data'] = {'priority': {}, 'files': {}}
must_find_files = True
- self.pickled_data['stats'] = {'upload': 0L, 'download': 0L}
+ self.pickled_data['stats'] = {'upload': 0L, 'download': 0L, 'download_http': 0L}
self.pickled_data['version'] = 1
# Initialize the saved state it if it wasn't found
@@ -913,7 +913,7 @@
must_find_files = True
self.pickled_data = {}
self.pickled_data['resume data'] = {'priority': {}, 'files': {}}
- self.pickled_data['stats'] = {'upload': 0L, 'download': 0L}
+ self.pickled_data['stats'] = {'upload': 0L, 'download': 0L, 'download_http': 0L}
self.pickled_data['version'] = 1
enabled_files = []
@@ -1099,7 +1099,7 @@
"""
- self.downmeasure.update_rate(x)
+ self.downmeasure.update_http_rate(x)
self.ratemeasure.data_came_in(x)
self.downloader.external_data_received(x)
@@ -1139,7 +1139,7 @@
@return: whether the engines were started
"""
-
+
if self.doneflag.isSet():
return False
@@ -1162,9 +1162,13 @@
total_up = long(self.pickled_data['stats']['upload'])
total_down = long(self.pickled_data['stats']['download'])
+ # The total_http stat isn't in files created with debtorrent-0.1.8
+ total_http = 0L;
+ if self.pickled_data['stats'].has_key('download_http'):
+ total_http = long(self.pickled_data['stats']['download_http'])
self.upmeasure = Measure(self.config['max_rate_period'],
self.config['upload_rate_fudge'], saved_total = total_up)
- self.downmeasure = Measure(self.config['max_rate_period'], saved_total = total_down)
+ self.downmeasure = Measure(self.config['max_rate_period'], saved_total = total_down, http_total = total_http)
if ratelimiter:
self.ratelimiter = ratelimiter
@@ -1325,7 +1329,8 @@
if self.fileselector and self.started:
torrentdata['version'] = 1
torrentdata['stats'] = {'upload': self.upmeasure.get_total(),
- 'download': self.downmeasure.get_total()}
+ 'download': self.downmeasure.get_total(),
+ 'download_http': self.downmeasure.get_http_subtotal()}
if not self.failed:
self.fileselector.finish()
torrentdata['resume data'] = self.fileselector.pickle()
diff -rN -u old-debtorrent/DebTorrent/launchmanycore.py new-debtorrent/DebTorrent/launchmanycore.py
--- old-debtorrent/DebTorrent/launchmanycore.py 2008-06-30 00:31:03.000000000 +0100
+++ new-debtorrent/DebTorrent/launchmanycore.py 2008-06-30 00:31:03.000000000 +0100
@@ -1,5 +1,6 @@
# Written by John Hoffman
# Modified by Cameron Dale
+# Modified by Steve Cotton
# see LICENSE.txt for license information
#
# $Id: launchmanycore.py 389 2008-06-22 19:23:06Z camrdale-guest $
@@ -35,6 +36,7 @@
import logging
from DebTorrent.BT1.AptListener import AptListener
from DebTorrent.HTTPHandler import HTTPHandler
+from DebTorrent.BT1.Statistics import CookedStatistics
logger = logging.getLogger('DebTorrent.launchmanycore')
@@ -413,20 +415,8 @@
def gather_stats(self):
"""Gather the statistics for the currently running torrents.
- Returns a list, one per running torrent, of tuples:
-
- (C{string}, C{string}, C{string}, C{string},
- C{int}, C{int}, C{string}, C{float},
- C{float}, C{float}, C{long}, C{long},
- C{long}, C{float}, C{string})
-
- Which are the name, info hash, current status, progress report,
- number of peers, number of seeds, seed message, number of distributed copies,
- upload rate, download rate, amount uploaded, amount downloaded,
- total length, time remaining, and latest error message.
-
- @rtype: C{list}
- @return: various statistics for the running torrents
+ @rtype: C{list} of C{CookedStatistics}
+ @return: A CookedStatistics for each of the running torrents
"""
@@ -438,60 +428,50 @@
name = cache['path']
else:
name = cache['name']
- size = 0
+
+ cooked = CookedStatistics(name, id)
d = self.downloads[hash]
- progress = '0.0%'
- peers = 0
- seeds = 0
- seedsmsg = "S"
- dist = 0.0
- uprate = 0.0
- dnrate = 0.0
- upamt = 0
- dnamt = 0
- t = 0
if d.is_dead():
- status = 'stopped'
+ cooked.status = 'stopped'
elif d.waiting:
- status = 'waiting for hash check'
+ cooked.status = 'waiting for hash check'
elif d.checking:
- status = d.status_msg
- progress = '%.1f%%' % (d.status_done*100)
+ cooked.status = d.status_msg
+ cooked.progress = '%.1f%%' % (d.status_done*100)
else:
stats = d.statsfunc()
s = stats['stats']
if d.seed:
- status = 'seeding'
- progress = '100.0%'
- seeds = s.numOldSeeds
- seedsmsg = "s"
- dist = s.numCopies
+ cooked.status = 'seeding'
+ cooked.progress = '100.0%'
+ cooked.numSeeds = s.numOldSeeds
+ cooked.numCopies = s.numCopies
else:
if s.numSeeds + s.numPeers:
- t = stats['time']
- if t == 0: # unlikely
- t = 0.01
- status = fmttime(t)
+ cooked.timeRemaining = stats['time']
+ if cooked.timeRemaining == 0: # unlikely
+ cooked.timeRemaining = 0.01
+ cooked.status = fmttime(cooked.timeRemaining)
else:
- t = -1
- status = 'connecting to peers'
- progress = '%.1f%%' % (int(stats['frac']*1000)/10.0)
- seeds = s.numSeeds
- dist = s.numCopies2
- dnrate = stats['down']
- peers = s.numPeers
- uprate = stats['up']
- upamt = s.upTotal
- dnamt = s.downTotal
- size = stats['wanted']
+ cooked.timeRemaining = -1
+ cooked.status = 'connecting to peers'
+ cooked.progress = '%.1f%%' % (int(stats['frac']*1000)/10.0)
+ cooked.numSeeds = s.numSeeds
+ cooked.numCopies = s.numCopies2
+ cooked.downRate = stats['down']
+ cooked.numPeers = s.numPeers
+ cooked.upRate = stats['up']
+ cooked.upTotal = s.upTotal
+ cooked.downTotal = s.downTotal
+ cooked.httpDownTotal = s.httpDownTotal
+ cooked.size = stats['wanted']
if d.is_dead() or d.status_errtime+300 > clock():
- msg = d.status_err[-1]
+ cooked.errorMessage = d.status_err[-1]
else:
- msg = ''
+ cooked.errorMessage = ''
- data.append(( name, id, status, progress, peers, seeds, seedsmsg,
- dist, uprate, dnrate, upamt, dnamt, size, t, msg ))
+ data.append(cooked)
return data
