src/gpodder/download.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # gPodder - A media aggregator and podcast client
   4 # Copyright (c) 2005-2008 Thomas Perl and the gPodder Team
   5 #
   6 # gPodder is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # gPodder is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20
  21 #
  22 #  download.py -- Download client using DownloadStatusManager
  23 #  Thomas Perl <thp@perli.net>   2007-09-15
  24 #
  25 #  Based on libwget.py (2005-10-29)
  26 #
  27
  28 from gpodder.liblogger import log
  29 from gpodder.libgpodder import gl
  30 from gpodder import util
  31 from gpodder import services
  32 from gpodder import resolver
  33 import gpodder
  34
  35 import threading
  36 import urllib
  37 import shutil
  38 import os.path
  39 import os
  40 import time
  41
  42 from xml.sax import saxutils
  43
  44 class DownloadCancelledException(Exception): pass
  45
  46 class gPodderDownloadHTTPError(Exception):
  47     def __init__(self, url, error_code, error_message):
  48         self.url = url
  49         self.error_code = error_code
  50         self.error_message = error_message
  51
  52 class DownloadURLOpener(urllib.FancyURLopener):
  53     version = gpodder.user_agent
  54
  55     def __init__( self, channel):
  56         if gl.config.proxy_use_environment:
  57             proxies = None
  58         else:
  59             proxies = {}
  60             if gl.config.http_proxy:
  61                 proxies['http'] = gl.config.http_proxy
  62             if gl.config.ftp_proxy:
  63                 proxies['ftp'] = gl.config.ftp_proxy
  64
  65         self.channel = channel
  66         urllib.FancyURLopener.__init__( self, proxies)
  67
  68     def http_error_default(self, url, fp, errcode, errmsg, headers):
  69         """
  70         FancyURLopener by default does not raise an exception when
  71         there is some unknown HTTP error code. We want to override
  72         this and provide a function to log the error and raise an
  73         exception, so we don't download the HTTP error page here.
  74         """
  75         # The following two lines are copied from urllib.URLopener's
  76         # implementation of http_error_default
  77         void = fp.read()
  78         fp.close()
  79         raise gPodderDownloadHTTPError(url, errcode, errmsg)
  80
  81     def prompt_user_passwd( self, host, realm):
  82         if self.channel.username or self.channel.password:
  83             log( 'Authenticating as "%s" to "%s" for realm "%s".', self.channel.username, host, realm, sender = self)
  84             return ( self.channel.username, self.channel.password )
  85
  86         return ( None, None )
  87
  88
  89 class DownloadThread(threading.Thread):
  90     MAX_UPDATES_PER_SEC = 1
  91
  92     def __init__( self, channel, episode, notification = None):
  93         threading.Thread.__init__( self)
  94         self.setDaemon( True)
  95
  96         self.channel = channel
  97         self.episode = episode
  98
  99         self.notification = notification
 100
 101         self.url = self.episode.url
 102         self.filename = self.episode.local_filename()
 103         self.tempname = os.path.join( os.path.dirname( self.filename), '.tmp-' + os.path.basename( self.filename))
 104
 105         # Make an educated guess about the total file size
 106         self.total_size = self.episode.length
 107
 108         self.cancelled = False
 109         self.start_time = 0.0
 110         self.speed = _('Queued')
 111         self.speed_value = 0
 112         self.progress = 0.0
 113         self.downloader = DownloadURLOpener( self.channel)
 114         self.last_update = 0.0
 115
 116         # Keep a copy of these global variables for comparison later
 117         self.limit_rate_value = gl.config.limit_rate_value
 118         self.limit_rate = gl.config.limit_rate
 119         self.start_blocks = 0
 120
 121     def cancel( self):
 122         self.cancelled = True
 123
 124     def status_updated( self, count, blockSize, totalSize):
 125         if totalSize:
 126             # We see a different "total size" while downloading,
 127             # so correct the total size variable in the thread
 128             if totalSize != self.total_size and totalSize > 0:
 129                 log('Correcting file size for %s from %d to %d while downloading.', self.url, self.total_size, totalSize, sender=self)
 130                 self.total_size = totalSize
 131             elif totalSize < 0:
 132                 # The current download has a negative value, so assume
 133                 # the total size given from the feed is correct
 134                 totalSize = self.total_size
 135             self.progress = 100.0*float(count*blockSize)/float(totalSize)
 136         else:
 137             self.progress = 100.0
 138
 139         # Sanity checks for "progress" in valid range (0..100)
 140         if self.progress < 0.0:
 141             log('Warning: Progress is lower than 0 (count=%d, blockSize=%d, totalSize=%d)', count, blockSize, totalSize, sender=self)
 142             self.progress = 0.0
 143         elif self.progress > 100.0:
 144             log('Warning: Progress is more than 100 (count=%d, blockSize=%d, totalSize=%d)', count, blockSize, totalSize, sender=self)
 145             self.progress = 100.0
 146
 147         self.calculate_speed( count, blockSize)
 148         if self.last_update < time.time() - (1.0 / self.MAX_UPDATES_PER_SEC):
 149             services.download_status_manager.update_status( self.download_id, speed = self.speed, progress = self.progress)
 150             self.last_update = time.time()
 151
 152         if self.cancelled:
 153             util.delete_file( self.tempname)
 154             raise DownloadCancelledException()
 155
 156     def calculate_speed( self, count, blockSize):
 157         if count % 5 == 0:
 158             now = time.time()
 159             if self.start_time > 0:
 160
 161                 # Has rate limiting been enabled or disabled?
 162                 if self.limit_rate != gl.config.limit_rate:
 163                     # If it has been enabled then reset base time and block count
 164                     if gl.config.limit_rate:
 165                         self.start_time = now
 166                         self.start_blocks = count
 167                     self.limit_rate = gl.config.limit_rate
 168
 169                 # Has the rate been changed and are we currently limiting?
 170                 if self.limit_rate_value != gl.config.limit_rate_value and self.limit_rate:
 171                     self.start_time = now
 172                     self.start_blocks = count
 173                     self.limit_rate_value = gl.config.limit_rate_value
 174
 175                 passed = now - self.start_time
 176                 if passed > 0:
 177                     speed = ((count-self.start_blocks)*blockSize)/passed
 178                 else:
 179                     speed = 0
 180             else:
 181                 self.start_time = now
 182                 self.start_blocks = count
 183                 passed = now - self.start_time
 184                 speed = count*blockSize
 185
 186             self.speed = '%s/s' % gl.format_filesize(speed)
 187             self.speed_value = speed
 188
 189             if gl.config.limit_rate and speed > gl.config.limit_rate_value:
 190                 # calculate the time that should have passed to reach
 191                 # the desired download rate and wait if necessary
 192                 should_have_passed = float((count-self.start_blocks)*blockSize)/(gl.config.limit_rate_value*1024.0)
 193                 if should_have_passed > passed:
 194                     # sleep a maximum of 10 seconds to not cause time-outs
 195                     delay = min( 10.0, float(should_have_passed-passed))
 196                     time.sleep( delay)
 197
 198     def run( self):
 199         self.download_id = services.download_status_manager.reserve_download_id()
 200         services.download_status_manager.register_download_id( self.download_id, self)
 201
 202         # Initial status update
 203         services.download_status_manager.update_status( self.download_id, episode = self.episode.title, url = self.episode.url, speed = self.speed, progress = self.progress)
 204
 205         acquired = services.download_status_manager.s_acquire()
 206         try:
 207             try:
 208                 if self.cancelled:
 209                     return
 210
 211                 util.delete_file( self.tempname)
 212                 (unused, headers) = self.downloader.retrieve( resolver.get_real_download_url(self.url), self.tempname, reporthook = self.status_updated)
 213
 214                 if 'content-type' in headers and headers['content-type'] != self.episode.mimetype:
 215                     log('Correcting mime type: %s => %s', self.episode.mimetype, headers['content-type'])
 216                     self.episode.mimetype = headers['content-type']
 217                     # File names are constructed with regard to the mime type.
 218                     self.filename = self.episode.local_filename()
 219
 220                 shutil.move( self.tempname, self.filename)
 221                 # Get the _real_ filesize once we actually have the file
 222                 self.episode.length = os.path.getsize(self.filename)
 223                 self.channel.addDownloadedItem( self.episode)
 224                 services.download_status_manager.download_completed(self.download_id)
 225
 226                 # If a user command has been defined, execute the command setting some environment variables
 227                 if len(gl.config.cmd_download_complete) > 0:
 228                     os.environ["GPODDER_EPISODE_URL"]=self.episode.url or ''
 229                     os.environ["GPODDER_EPISODE_TITLE"]=self.episode.title or ''
 230                     os.environ["GPODDER_EPISODE_FILENAME"]=self.filename or ''
 231                     os.environ["GPODDER_EPISODE_PUBDATE"]=str(int(self.episode.pubDate))
 232                     os.environ["GPODDER_EPISODE_LINK"]=self.episode.link or ''
 233                     os.environ["GPODDER_EPISODE_DESC"]=self.episode.description or ''
 234                     threading.Thread(target=gl.ext_command_thread, args=(self.notification,gl.config.cmd_download_complete)).start()
 235
 236             finally:
 237                 services.download_status_manager.remove_download_id( self.download_id)
 238                 services.download_status_manager.s_release( acquired)
 239         except DownloadCancelledException:
 240             log('Download has been cancelled: %s', self.episode.title, traceback=None, sender=self)
 241         except IOError, ioe:
 242             if self.notification is not None:
 243                 title = ioe.strerror
 244                 message = _('An error happened while trying to download <b>%s</b>.') % ( saxutils.escape( self.episode.title), )
 245                 self.notification( message, title)
 246             log( 'Error "%s" while downloading "%s": %s', ioe.strerror, self.episode.title, ioe.filename, sender = self)
 247         except gPodderDownloadHTTPError, gdhe:
 248             if self.notification is not None:
 249                 title = gdhe.error_message
 250                 message = _('An error (HTTP %d) happened while trying to download <b>%s</b>.') % ( gdhe.error_code, saxutils.escape( self.episode.title), )
 251                 self.notification( message, title)
 252             log( 'HTTP error %s while downloading "%s": %s', gdhe.error_code, self.episode.title, gdhe.error_message, sender=self)
 253         except:
 254             log( 'Error while downloading "%s".', self.episode.title, sender = self, traceback = True)
 255