feedlint

   1 #!/usr/bin/env python
   2
   3 from optparse import OptionParser
   4 import sys, shutil, tempfile, urlparse
   5 import socket
   6 import urllib2, os, httplib
   7 import ftplib
   8 import logging, time, traceback
   9 from logging import info
  10
  11 from zeroinstall import SafeException
  12 from zeroinstall.injector import model, gpg, namespaces, qdom
  13
  14 from display import checking, result, error, highlight, error_new_line
  15
  16 now = time.time()
  17
  18 version = '0.7'
  19
  20 WEEK = 60 * 60 * 24 * 7
  21
  22 def host(address):
  23         if hasattr(address, 'hostname'):
  24                 return address.hostname
  25         else:
  26                 return address[1].split(':', 1)[0]
  27
  28 def port(address):
  29         if hasattr(address, 'port'):
  30                 return address.port
  31         else:
  32                 port = address[1].split(':', 1)[1:]
  33                 if port:
  34                         return int(port[0])
  35                 else:
  36                         return None
  37
  38 assert port(('http', 'foo:81')) == 81
  39 assert port(urlparse.urlparse('http://foo:81')) == 81
  40
  41 parser = OptionParser(usage="usage: %prog [options] feed.xml")
  42 parser.add_option("-d", "--dependencies", help="also check feeds for dependencies", action='store_true')
  43 parser.add_option("-s", "--skip-archives", help="don't check the archives are OK", action='store_true')
  44 parser.add_option("-v", "--verbose", help="more verbose output", action='count')
  45 parser.add_option("-V", "--version", help="display version information", action='store_true')
  46
  47 (options, args) = parser.parse_args()
  48
  49 if options.version:
  50         print "FeedLint (zero-install) " + version
  51         print "Copyright (C) 2007 Thomas Leonard"
  52         print "This program comes with ABSOLUTELY NO WARRANTY,"
  53         print "to the extent permitted by law."
  54         print "You may redistribute copies of this program"
  55         print "under the terms of the GNU General Public License."
  56         print "For more information about these matters, see the file named COPYING."
  57         sys.exit(0)
  58
  59 if options.verbose:
  60         logger = logging.getLogger()
  61         if options.verbose == 1:
  62                 logger.setLevel(logging.INFO)
  63         else:
  64                 logger.setLevel(logging.DEBUG)
  65
  66 if len(args) < 1:
  67         parser.print_help()
  68         sys.exit(1)
  69
  70 checked = set()
  71
  72 try:
  73         to_check = [model.canonical_iface_uri(a) for a in args]
  74 except SafeException, ex:
  75         if options.verbose: raise
  76         print >>sys.stderr, ex
  77         sys.exit(1)
  78
  79 def check_key(feed_url, fingerprint):
  80         for line in os.popen('gpg --with-colons --list-keys %s' % fingerprint):
  81                 if line.startswith('pub:'):
  82                         key_id = line.split(':')[4]
  83                         break
  84         else:
  85                 raise SafeException('Failed to find key with fingerprint %s on your keyring' % fingerprint)
  86
  87         key_url = urlparse.urljoin(feed_url, '%s.gpg' % key_id)
  88
  89         if key_url in checked:
  90                 info("(already checked key URL %s)", key_url)
  91         else:
  92                 checking("Checking key %s" % key_url)
  93                 urllib2.urlopen(key_url).read()
  94                 result('OK')
  95                 checked.add(key_url)
  96
  97 def get_http_size(url, ttl = 3):
  98         address = urlparse.urlparse(url)
  99
 100         if url.lower().startswith('http://'):
 101                 http = httplib.HTTPConnection(host(address), port(address) or 80)
 102         elif url.lower().startswith('https://'):
 103                 http = httplib.HTTPSConnection(host(address), port(address) or 443)
 104         else:
 105                 assert False, url
 106
 107         parts = url.split('/', 3)
 108         if len(parts) == 4:
 109                 path = parts[3]
 110         else:
 111                 path = ''
 112
 113         http.request('HEAD', '/' + path, headers = {'Host': host(address)})
 114         response = http.getresponse()
 115         try:
 116                 if response.status == 200:
 117                         return response.getheader('Content-Length')
 118                 elif response.status in (301, 302, 303):
 119                         new_url_rel = response.getheader('Location') or response.getheader('URI')
 120                         new_url = urlparse.urljoin(url, new_url_rel)
 121                 else:
 122                         raise SafeException("HTTP error: got status code %s" % response.status)
 123         finally:
 124                 response.close()
 125
 126         if ttl:
 127                 result("Moved", 'YELLOW')
 128                 checking("Checking new URL %s" % new_url)
 129                 assert new_url
 130                 return get_http_size(new_url, ttl - 1)
 131         else:
 132                 raise SafeException('Too many redirections.')
 133
 134 def get_ftp_size(url):
 135         address = urlparse.urlparse(url)
 136         ftp = ftplib.FTP(host(address))
 137         try:
 138                 ftp.login()
 139                 ftp.voidcmd('TYPE I')
 140                 return ftp.size(url.split('/', 3)[3])
 141         finally:
 142                 ftp.close()
 143
 144 def get_size(url):
 145         scheme = urlparse.urlparse(url)[0].lower()
 146         if scheme.startswith('http') or scheme.startswith('https'):
 147                 return get_http_size(url)
 148         elif scheme.startswith('ftp'):
 149                 return get_ftp_size(url)
 150         else:
 151                 raise SafeException("Unknown scheme '%s' in '%s'" % (scheme, url))
 152
 153 def check_source(source):
 154         if hasattr(source, 'url'):
 155                 checking("Checking archive %s" % source.url)
 156                 actual_size = get_size(source.url)
 157                 if actual_size is None:
 158                         result("No Content-Length for archive; can't check", 'YELLOW')
 159                 else:
 160                         actual_size = int(actual_size)
 161                         expected_size = source.size + (source.start_offset or 0)
 162                         if actual_size != expected_size:
 163                                 error('Bad length')
 164                                 raise SafeException("Expected archive to have a size of %d, but server says it is %d" %
 165                                                 (expected_size, actual_size))
 166                         result('OK')
 167         elif hasattr(source, 'steps'):
 168                 for step in source.steps:
 169                         check_source(step)
 170
 171 existing_urls = set()
 172 def check_exists(url):
 173         if url in existing_urls: return         # Already checked
 174
 175         checking("Checking URL exists %s" % url)
 176         get_size(url)
 177         result('OK')
 178         existing_urls.add(url)
 179
 180 n_errors = 0
 181
 182 def check_gpg_sig(feed_url, stream):
 183         start = stream.read(5)
 184         if start == '<?xml':
 185                 result('Fetched')
 186         elif start == '-----':
 187                 result('Old sig', colour = 'RED')
 188                 error_new_line('  Feed has an old-style plain GPG signature. Use 0publish --xmlsign.',
 189                                 colour = 'YELLOW')
 190         else:
 191                 result('Fetched')
 192                 error_new_line('  Unknown format. File starts "%s"' % start)
 193
 194         data, sigs = gpg.check_stream(stream)
 195
 196         for s in sigs:
 197                 if isinstance(s, gpg.ValidSig):
 198                         check_key(feed_url, s.fingerprint)
 199                 else:
 200                         raise SafeException("Can't check sig: %s" % s)
 201
 202         return data
 203
 204 while to_check:
 205         feed = to_check.pop()
 206         if feed in checked:
 207                 info("Already checked feed %s", feed)
 208                 continue
 209
 210         checked.add(feed)
 211
 212         checking("Checking " + feed, indent = 0)
 213
 214         try:
 215                 if feed.startswith('/'):
 216                         with open(feed) as stream:
 217                                 root = qdom.parse(stream)
 218
 219                                 if "uri" in root.attrs:
 220                                         stream.seek(0)
 221                                         try:
 222                                                 check_gpg_sig(root.attrs['uri'], stream)
 223                                         except SafeException, ex:
 224                                                 n_errors += 1
 225                                                 error_new_line('  %s' % ex)
 226
 227                                 feed_obj = model.ZeroInstallFeed(root, local_path = feed)
 228                         result('Local')
 229                 else:
 230                         tmp = tempfile.TemporaryFile(prefix = 'feedlint-')
 231                         try:
 232                                 try:
 233                                         stream = urllib2.urlopen(feed)
 234                                         shutil.copyfileobj(stream, tmp)
 235                                 except Exception as ex:
 236                                         raise SafeException('Failed to fetch feed: {ex}'.format(ex = ex))
 237                                 tmp.seek(0)
 238
 239                                 data = check_gpg_sig(feed, tmp)
 240                                 tmp.seek(0)
 241
 242                                 feed_obj = model.ZeroInstallFeed(qdom.parse(data))
 243
 244                                 if feed_obj.url != feed:
 245                                         raise SafeException('Incorrect URL "%s"' % feed_obj.url)
 246
 247                         finally:
 248                                 tmp.close()
 249
 250                 for f in feed_obj.feeds:
 251                         info("Will check feed %s", f.uri)
 252                         to_check.append(f.uri)
 253
 254                 highest_version = None
 255                 for impl in feed_obj.implementations.values():
 256                         if hasattr(impl, 'dependencies'):
 257                                 for r in impl.dependencies.values():
 258                                         if r.interface not in checked:
 259                                                 info("Will check dependency %s", r)
 260                                                 if options.dependencies:
 261                                                         to_check.append(r.interface)
 262                                                 else:
 263                                                         check_exists(r.interface)
 264                         if hasattr(impl, 'download_sources') and not options.skip_archives:
 265                                 for source in impl.download_sources:
 266                                         check_source(source)
 267                         if impl.local_path is None:
 268                                 if not impl.digests:
 269                                         raise SafeException("Version {version} has no digests".format(version = impl.get_version()))
 270                         stability = impl.upstream_stability or model.testing
 271                         if highest_version is None or impl.version > highest_version.version:
 272                                 highest_version = impl
 273                         if stability == model.testing:
 274                                 testing_error = None
 275                                 if not impl.released:
 276                                         if not impl.local_path:
 277                                                 testing_error = "No release date on testing version"
 278                                 else:
 279                                         try:
 280                                                 released = time.strptime(impl.released, '%Y-%m-%d')
 281                                         except ValueError, ex:
 282                                                 testing_error = "Can't parse date"
 283                                         else:
 284                                                 ago = now - time.mktime(released)
 285                                                 if ago < 0:
 286                                                         testing_error = 'Release date is in the future!'
 287                                 if testing_error:
 288                                         raise SafeException("Version %s: %s (released %s)" % (impl.get_version(), testing_error, impl.released))
 289
 290                         # Old Windows versions use 32-bit integers to store versions. Newer versions use 64-bit ones, but in general
 291                         # keeping the numbers small is helpful.
 292                         for i in range(0, len(impl.version), 2):
 293                                 for x in impl.version[i]:
 294                                         if x >= (1 << 31):
 295                                                 raise SafeException("Version %s: component %s won't fit in a 32-bit signed integer" % (impl.get_version(), x))
 296
 297                 if highest_version and (highest_version.upstream_stability or model.testing) is model.testing:
 298                         print highlight('  Highest version (%s) is still "testing"' % highest_version.get_version(), 'YELLOW')
 299
 300                 for homepage in feed_obj.get_metadata(namespaces.XMLNS_IFACE, 'homepage'):
 301                         check_exists(homepage.content)
 302
 303                 for icon in feed_obj.get_metadata(namespaces.XMLNS_IFACE, 'icon'):
 304                         check_exists(icon.getAttribute('href'))
 305
 306         except (urllib2.HTTPError, httplib.BadStatusLine, socket.error, ftplib.error_perm), ex:
 307                 err_msg = str(ex).strip() or str(type(ex))
 308                 error_new_line('  ' + err_msg)
 309                 n_errors += 1
 310                 if options.verbose: traceback.print_exc()
 311         except SafeException, ex:
 312                 if options.verbose: raise
 313                 error_new_line('  ' + str(ex))
 314                 n_errors += 1
 315
 316 if n_errors == 0:
 317         print "OK"
 318 else:
 319         print "\nERRORS FOUND:", n_errors
 320         sys.exit(1)