zeroinstall/injector/download.py

   1 """
   2 Handles URL downloads.
   3
   4 This is the low-level interface for downloading interfaces, implementations, icons, etc.
   5
   6 @see: L{fetch} higher-level API for downloads that uses this module
   7 """
   8
   9 # Copyright (C) 2009, Thomas Leonard
  10 # See the README file for details, or visit http://0install.net.
  11
  12 import tempfile, os, sys, subprocess
  13 from zeroinstall import SafeException
  14 from zeroinstall.support import tasks
  15 from logging import info, debug
  16 from zeroinstall import _
  17
  18 download_starting = "starting"  # Waiting for UI to start it
  19 download_fetching = "fetching"  # In progress
  20 download_complete = "complete"  # Downloaded and cached OK
  21 download_failed = "failed"
  22
  23 RESULT_OK = 0
  24 RESULT_FAILED = 1
  25 RESULT_NOT_MODIFIED = 2
  26
  27 class DownloadError(SafeException):
  28         """Download process failed."""
  29         pass
  30
  31 class DownloadAborted(DownloadError):
  32         """Download aborted because of a call to L{Download.abort}"""
  33         def __init__(self, message):
  34                 SafeException.__init__(self, message or _("Download aborted at user's request"))
  35
  36 class Download(object):
  37         """A download of a single resource to a temporary file.
  38         @ivar url: the URL of the resource being fetched
  39         @type url: str
  40         @ivar tempfile: the file storing the downloaded data
  41         @type tempfile: file
  42         @ivar status: the status of the download
  43         @type status: (download_starting | download_fetching | download_failed | download_complete)
  44         @ivar errors: data received from the child's stderr
  45         @type errors: str
  46         @ivar expected_size: the expected final size of the file
  47         @type expected_size: int | None
  48         @ivar downloaded: triggered when the download ends (on success or failure)
  49         @type downloaded: L{tasks.Blocker}
  50         @ivar hint: hint passed by and for caller
  51         @type hint: object
  52         @ivar child: the child process
  53         @type child: subprocess.Popen
  54         @ivar aborted_by_user: whether anyone has called L{abort}
  55         @type aborted_by_user: bool
  56         @ivar unmodified: whether the resource was not modified since the modification_time given at construction
  57         @type unmodified: bool
  58         """
  59         __slots__ = ['url', 'tempfile', 'status', 'errors', 'expected_size', 'downloaded',
  60                      'hint', 'child', '_final_total_size', 'aborted_by_user',
  61                      'modification_time', 'unmodified']
  62
  63         def __init__(self, url, hint = None, modification_time = None):
  64                 """Create a new download object.
  65                 @param url: the resource to download
  66                 @param hint: object with which this download is associated (an optional hint for the GUI)
  67                 @param modification_time: string with HTTP date that indicates last modification time.
  68                   The resource will not be downloaded if it was not modified since that date.
  69                 @postcondition: L{status} == L{download_starting}."""
  70                 self.url = url
  71                 self.status = download_starting
  72                 self.hint = hint
  73                 self.aborted_by_user = False
  74                 self.modification_time = modification_time
  75                 self.unmodified = False
  76
  77                 self.tempfile = None            # Stream for result
  78                 self.errors = None
  79                 self.downloaded = None
  80
  81                 self.expected_size = None       # Final size (excluding skipped bytes)
  82                 self._final_total_size = None   # Set when download is finished
  83
  84                 self.child = None
  85
  86         def start(self):
  87                 """Create a temporary file and begin the download.
  88                 @precondition: L{status} == L{download_starting}"""
  89                 assert self.status == download_starting
  90                 assert self.downloaded is None
  91
  92                 self.tempfile = tempfile.TemporaryFile(prefix = 'injector-dl-data-')
  93
  94                 task = tasks.Task(self._do_download(), "download " + self.url)
  95                 self.downloaded = task.finished
  96
  97         def _do_download(self):
  98                 """Will trigger L{downloaded} when done (on success or failure)."""
  99                 self.errors = ''
 100
 101                 # Can't use fork here, because Windows doesn't have it
 102                 assert self.child is None, self.child
 103                 child_args = [sys.executable, '-u', __file__, self.url]
 104                 if self.modification_time: child_args.append(self.modification_time)
 105                 self.child = subprocess.Popen(child_args, stderr = subprocess.PIPE, stdout = self.tempfile)
 106
 107                 self.status = download_fetching
 108
 109                 # Wait for child to exit, collecting error output as we go
 110
 111                 while True:
 112                         yield tasks.InputBlocker(self.child.stderr, "read data from " + self.url)
 113
 114                         data = os.read(self.child.stderr.fileno(), 100)
 115                         if not data:
 116                                 break
 117                         self.errors += data
 118
 119                 # Download is complete...
 120
 121                 assert self.status is download_fetching
 122                 assert self.tempfile is not None
 123                 assert self.child is not None
 124
 125                 status = self.child.wait()
 126                 self.child = None
 127
 128                 errors = self.errors
 129                 self.errors = None
 130
 131                 if status == RESULT_NOT_MODIFIED:
 132                         debug("%s not modified", self.url)
 133                         self.unmodified = True
 134                         self.status = download_complete
 135                         self._final_total_size = 0
 136                         self.downloaded.trigger()
 137                         return
 138
 139                 if status and not self.aborted_by_user and not errors:
 140                         errors = _('Download process exited with error status '
 141                                            'code %s') % hex(status)
 142
 143                 self._final_total_size = self.get_bytes_downloaded_so_far()
 144
 145                 stream = self.tempfile
 146                 self.tempfile = None
 147
 148                 try:
 149                         if self.aborted_by_user:
 150                                 raise DownloadAborted(errors)
 151
 152                         if errors:
 153                                 raise DownloadError(errors.strip())
 154
 155                         # Check that the download has the correct size, if we know what it should be.
 156                         if self.expected_size is not None:
 157                                 size = os.fstat(stream.fileno()).st_size
 158                                 if size != self.expected_size:
 159                                         raise SafeException(_('Downloaded archive has incorrect size.\n'
 160                                                         'URL: %(url)s\n'
 161                                                         'Expected: %(expected_size)d bytes\n'
 162                                                         'Received: %(size)d bytes') % {'url': self.url, 'expected_size': self.expected_size, 'size': size})
 163                 except:
 164                         self.status = download_failed
 165                         _, ex, tb = sys.exc_info()
 166                         self.downloaded.trigger(exception = (ex, tb))
 167                 else:
 168                         self.status = download_complete
 169                         self.downloaded.trigger()
 170
 171         def abort(self):
 172                 """Signal the current download to stop.
 173                 @postcondition: L{aborted_by_user}"""
 174                 if self.child is not None:
 175                         info(_("Killing download process %s"), self.child.pid)
 176                         import signal
 177                         os.kill(self.child.pid, signal.SIGTERM)
 178                         self.aborted_by_user = True
 179                 else:
 180                         self.status = download_failed
 181
 182         def get_current_fraction(self):
 183                 """Returns the current fraction of this download that has been fetched (from 0 to 1),
 184                 or None if the total size isn't known.
 185                 @return: fraction downloaded
 186                 @rtype: int | None"""
 187                 if self.status is download_starting:
 188                         return 0
 189                 if self.tempfile is None:
 190                         return 1
 191                 if self.expected_size is None:
 192                         return None             # Unknown
 193                 current_size = self.get_bytes_downloaded_so_far()
 194                 return float(current_size) / self.expected_size
 195
 196         def get_bytes_downloaded_so_far(self):
 197                 """Get the download progress. Will be zero if the download has not yet started.
 198                 @rtype: int"""
 199                 if self.status is download_starting:
 200                         return 0
 201                 elif self.status is download_fetching:
 202                         return os.fstat(self.tempfile.fileno()).st_size
 203                 else:
 204                         return self._final_total_size
 205
 206         def __str__(self):
 207                 return _("<Download from %s>") % self.url
 208
 209 if __name__ == '__main__':
 210         def _download_as_child(url, if_modified_since):
 211                 from httplib import HTTPException
 212                 from urllib2 import urlopen, Request, HTTPError, URLError
 213                 try:
 214                         #print "Child downloading", url
 215                         if url.startswith('/'):
 216                                 if not os.path.isfile(url):
 217                                         print >>sys.stderr, "File '%s' does not " \
 218                                                 "exist!" % url
 219                                         return
 220                                 src = file(url)
 221                         elif url.startswith('http:') or url.startswith('ftp:'):
 222                                 req = Request(url)
 223                                 if url.startswith('http:') and if_modified_since:
 224                                         req.add_header('If-Modified-Since', if_modified_since)
 225                                 src = urlopen(req)
 226                         else:
 227                                 raise Exception(_('Unsupported URL protocol in: %s') % url)
 228
 229                         try:
 230                                 sock = src.fp._sock
 231                         except AttributeError:
 232                                 sock = src.fp.fp._sock  # Python 2.5 on FreeBSD
 233                         while True:
 234                                 data = sock.recv(256)
 235                                 if not data: break
 236                                 os.write(1, data)
 237
 238                         sys.exit(RESULT_OK)
 239                 except (HTTPError, URLError, HTTPException), ex:
 240                         if isinstance(ex, HTTPError) and ex.code == 304: # Not modified
 241                                 sys.exit(RESULT_NOT_MODIFIED)
 242                         print >>sys.stderr, "Error downloading '" + url + "': " + (str(ex) or str(ex.__class__.__name__))
 243                         sys.exit(RESULT_FAILED)
 244         assert (len(sys.argv) == 2) or (len(sys.argv) == 3), "Usage: download URL [If-Modified-Since-Date], not %s" % sys.argv
 245         if len(sys.argv) >= 3:
 246                 if_modified_since_date = sys.argv[2]
 247         else:
 248                 if_modified_since_date = None
 249         _download_as_child(sys.argv[1], if_modified_since_date)