feedlint

   1 #!/usr/bin/env python
   2
   3 from optparse import OptionParser
   4 import sys, shutil, tempfile, urlparse
   5 import socket
   6 import urllib2, os, httplib
   7 import ftplib
   8 import logging, time, traceback
   9 from logging import info
  10
  11 from zeroinstall import SafeException
  12 from zeroinstall.support import basedir, tasks
  13 from zeroinstall.injector import model, gpg, namespaces, qdom
  14 from zeroinstall.injector.config import load_config
  15
  16 from display import checking, result, error, highlight, error_new_line
  17
  18 config = load_config()
  19
  20 now = time.time()
  21
  22 version = '0.9'
  23
  24 WEEK = 60 * 60 * 24 * 7
  25
  26 def host(address):
  27         if hasattr(address, 'hostname'):
  28                 return address.hostname
  29         else:
  30                 return address[1].split(':', 1)[0]
  31
  32 def port(address):
  33         if hasattr(address, 'port'):
  34                 return address.port
  35         else:
  36                 port = address[1].split(':', 1)[1:]
  37                 if port:
  38                         return int(port[0])
  39                 else:
  40                         return None
  41
  42 assert port(('http', 'foo:81')) == 81
  43 assert port(urlparse.urlparse('http://foo:81')) == 81
  44
  45 parser = OptionParser(usage="usage: %prog [options] feed.xml")
  46 parser.add_option("-d", "--dependencies", help="also check feeds for dependencies", action='store_true')
  47 parser.add_option("-o", "--offline", help="only perform offline checks", action='store_true')
  48 parser.add_option("-s", "--skip-archives", help="don't check the archives are OK", action='store_true')
  49 parser.add_option("-v", "--verbose", help="more verbose output", action='count')
  50 parser.add_option("-V", "--version", help="display version information", action='store_true')
  51
  52 (options, args) = parser.parse_args()
  53
  54 if options.version:
  55         print "FeedLint (zero-install) " + version
  56         print "Copyright (C) 2007 Thomas Leonard"
  57         print "This program comes with ABSOLUTELY NO WARRANTY,"
  58         print "to the extent permitted by law."
  59         print "You may redistribute copies of this program"
  60         print "under the terms of the GNU General Public License."
  61         print "For more information about these matters, see the file named COPYING."
  62         sys.exit(0)
  63
  64 if options.offline:
  65         config.network_use = model.network_offline
  66         # Catch bugs
  67         os.environ['http_proxy'] = 'http://localhost:9999/offline-mode'
  68
  69 if options.verbose:
  70         logger = logging.getLogger()
  71         if options.verbose == 1:
  72                 logger.setLevel(logging.INFO)
  73         else:
  74                 logger.setLevel(logging.DEBUG)
  75
  76 if len(args) < 1:
  77         parser.print_help()
  78         sys.exit(1)
  79
  80 checked = set()
  81
  82 def arg_to_uri(arg):
  83         app = config.app_mgr.lookup_app(arg, missing_ok = True)
  84         if app is not None:
  85                 return app.get_requirements().interface_uri
  86         else:
  87                 return model.canonical_iface_uri(a)
  88
  89 try:
  90         to_check = [arg_to_uri(a) for a in args]
  91 except SafeException, ex:
  92         if options.verbose: raise
  93         print >>sys.stderr, ex
  94         sys.exit(1)
  95
  96 def check_key(feed_url, keyid):
  97         for line in os.popen('gpg --with-colons --list-keys %s' % keyid):
  98                 if line.startswith('pub:'):
  99                         key_id = line.split(':')[4]
 100                         break
 101         else:
 102                 raise SafeException('Failed to find key "%s" on your keyring' % keyid)
 103
 104         if options.offline: return
 105
 106         key_url = urlparse.urljoin(feed_url, '%s.gpg' % key_id)
 107
 108         if key_url in checked:
 109                 info("(already checked key URL %s)", key_url)
 110         else:
 111                 checking("Checking key %s" % key_url)
 112                 urllib2.urlopen(key_url).read()
 113                 result('OK')
 114                 checked.add(key_url)
 115
 116 def get_http_size(url, ttl = 3):
 117         assert not options.offline
 118         address = urlparse.urlparse(url)
 119
 120         if url.lower().startswith('http://'):
 121                 http = httplib.HTTPConnection(host(address), port(address) or 80)
 122         elif url.lower().startswith('https://'):
 123                 http = httplib.HTTPSConnection(host(address), port(address) or 443)
 124         else:
 125                 assert False, url
 126
 127         parts = url.split('/', 3)
 128         if len(parts) == 4:
 129                 path = parts[3]
 130         else:
 131                 path = ''
 132
 133         http.request('HEAD', '/' + path, headers = {'Host': host(address)})
 134         response = http.getresponse()
 135         try:
 136                 if response.status == 200:
 137                         return response.getheader('Content-Length')
 138                 elif response.status in (301, 302, 303):
 139                         new_url_rel = response.getheader('Location') or response.getheader('URI')
 140                         new_url = urlparse.urljoin(url, new_url_rel)
 141                 else:
 142                         raise SafeException("HTTP error: got status code %s" % response.status)
 143         finally:
 144                 response.close()
 145
 146         if ttl:
 147                 result("Moved", 'YELLOW')
 148                 checking("Checking new URL %s" % new_url)
 149                 assert new_url
 150                 return get_http_size(new_url, ttl - 1)
 151         else:
 152                 raise SafeException('Too many redirections.')
 153
 154 def get_ftp_size(url):
 155         address = urlparse.urlparse(url)
 156         ftp = ftplib.FTP(host(address))
 157         try:
 158                 ftp.login()
 159                 ftp.voidcmd('TYPE I')
 160                 return ftp.size(url.split('/', 3)[3])
 161         finally:
 162                 ftp.close()
 163
 164 def get_size(url, base_url = None):
 165         if '://' not in url:
 166                 # Local path
 167                 if base_url and base_url.startswith('/'):
 168                         # Local feed; OK
 169                         local_path = os.path.join(os.path.dirname(base_url), url)
 170                         if not os.path.exists(local_path):
 171                                 raise SafeException("Local file '%s' does not exist (should be a URL?)" % url)
 172                         return os.path.getsize(local_path)
 173                 if base_url is not None:
 174                         raise SafeException("Local file reference '%s' in non-local feed '%s'" % (url, base_url))
 175                 # Fall-through to Unknown scheme error
 176
 177         scheme = urlparse.urlparse(url)[0].lower()
 178         if scheme.startswith('http') or scheme.startswith('https'):
 179                 return get_http_size(url)
 180         elif scheme.startswith('ftp'):
 181                 return get_ftp_size(url)
 182         else:
 183                 raise SafeException("Unknown scheme '%s' in '%s'" % (scheme, url))
 184
 185 def check_source(feed_url, source):
 186         if hasattr(source, 'url'):
 187                 checking("Checking archive %s" % source.url)
 188                 actual_size = get_size(source.url, feed_url)
 189                 if actual_size is None:
 190                         result("No Content-Length for archive; can't check", 'YELLOW')
 191                 else:
 192                         actual_size = int(actual_size)
 193                         expected_size = source.size + (source.start_offset or 0)
 194                         if actual_size != expected_size:
 195                                 error('Bad length')
 196                                 raise SafeException("Expected archive to have a size of %d, but server says it is %d" %
 197                                                 (expected_size, actual_size))
 198                         result('OK')
 199         elif hasattr(source, 'steps'):
 200                 for step in source.steps:
 201                         check_source(feed_url, step)
 202
 203 existing_urls = set()
 204 def check_exists(url):
 205         if url in existing_urls: return         # Already checked
 206         if options.offline: return
 207
 208         checking("Checking URL exists %s" % url)
 209         get_size(url)
 210         result('OK')
 211         existing_urls.add(url)
 212
 213 def scan_implementations(impls, dom):
 214         """Add each implementation in dom to impls. Error if duplicate."""
 215         for elem in dom.childNodes:
 216                 if elem.uri != namespaces.XMLNS_IFACE: continue
 217                 if elem.name == 'implementation':
 218                         impl_id = elem.attrs['id']
 219                         if impl_id in impls:
 220                                 raise SafeException("Duplicate ID {id}!".format(id = impl_id))
 221                         impls[impl_id] = elem
 222                 elif elem.name == 'group':
 223                         scan_implementations(impls, elem)
 224
 225 n_errors = 0
 226
 227 def check_gpg_sig(feed_url, stream):
 228         start = stream.read(5)
 229         if start == '<?xml':
 230                 result('Fetched')
 231         elif start == '-----':
 232                 result('Old sig', colour = 'RED')
 233                 error_new_line('  Feed has an old-style plain GPG signature. Use 0publish --xmlsign.',
 234                                 colour = 'YELLOW')
 235         else:
 236                 result('Fetched')
 237                 error_new_line('  Unknown format. File starts "%s"' % start)
 238
 239         data, sigs = gpg.check_stream(stream)
 240
 241         for s in sigs:
 242                 if isinstance(s, gpg.ValidSig):
 243                         check_key(feed_url, s.fingerprint)
 244                 elif isinstance(s, gpg.ErrSig) and s.need_key():
 245                         # Download missing key
 246                         key = s.need_key()
 247                         key_url = urlparse.urljoin(feed_url, '%s.gpg' % key)
 248                         dl = config.fetcher.download_url(key_url)
 249                         stream = dl.tempfile
 250                         tasks.wait_for_blocker(dl.downloaded)
 251
 252                         stream.seek(0)
 253                         gpg.import_key(stream)
 254                         stream.close()
 255
 256                         check_key(feed_url, key)
 257                 else:
 258                         raise SafeException("Can't check sig: %s" % s)
 259
 260         return data
 261
 262 while to_check:
 263         feed = to_check.pop()
 264         if feed in checked:
 265                 info("Already checked feed %s", feed)
 266                 continue
 267
 268         checked.add(feed)
 269
 270         checking("Checking " + feed, indent = 0)
 271         is_remote = feed.startswith('http://') or feed.startswith('https://')
 272
 273         try:
 274                 if not is_remote:
 275                         with open(feed) as stream:
 276                                 dom = qdom.parse(stream)
 277
 278                                 if "uri" in dom.attrs:
 279                                         stream.seek(0)
 280                                         try:
 281                                                 check_gpg_sig(dom.attrs['uri'], stream)
 282                                         except SafeException, ex:
 283                                                 n_errors += 1
 284                                                 error_new_line('  %s' % ex)
 285
 286                                 feed_obj = model.ZeroInstallFeed(dom, local_path = feed if "uri" not in dom.attrs else None)
 287                         result('Local')
 288                 elif options.offline:
 289                         cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(feed))
 290                         if not cached:
 291                                 raise SafeException('Not cached (offline-mode)')
 292                         with open(cached, 'rb') as stream:
 293                                 dom = qdom.parse(stream)
 294                         feed_obj = model.ZeroInstallFeed(dom)
 295                         result('Cached')
 296                 else:
 297                         tmp = tempfile.TemporaryFile(prefix = 'feedlint-')
 298                         try:
 299                                 try:
 300                                         stream = urllib2.urlopen(feed)
 301                                         shutil.copyfileobj(stream, tmp)
 302                                 except Exception as ex:
 303                                         raise SafeException('Failed to fetch feed: {ex}'.format(ex = ex))
 304                                 tmp.seek(0)
 305
 306                                 data = check_gpg_sig(feed, tmp)
 307                                 tmp.seek(0)
 308
 309                                 dom = qdom.parse(data)
 310                                 feed_obj = model.ZeroInstallFeed(dom)
 311
 312                                 if feed_obj.url != feed:
 313                                         raise SafeException('Incorrect URL "%s"' % feed_obj.url)
 314
 315                         finally:
 316                                 tmp.close()
 317
 318                 # Check for duplicate IDs
 319                 scan_implementations({}, dom)
 320
 321                 for f in feed_obj.feeds:
 322                         info("Will check feed %s", f.uri)
 323                         to_check.append(f.uri)
 324
 325                 highest_version = None
 326                 for impl in sorted(feed_obj.implementations.values()):
 327                         if hasattr(impl, 'dependencies'):
 328                                 for r in impl.dependencies.values():
 329                                         if r.interface not in checked:
 330                                                 info("Will check dependency %s", r)
 331                                                 if options.dependencies:
 332                                                         to_check.append(r.interface)
 333                                                 else:
 334                                                         check_exists(r.interface)
 335                         if hasattr(impl, 'download_sources') and not options.skip_archives:
 336                                 if not options.offline:
 337                                         for source in impl.download_sources:
 338                                                 check_source(feed_obj.url, source)
 339                         if impl.local_path is None:
 340                                 if not impl.digests:
 341                                         raise SafeException("Version {version} has no digests".format(version = impl.get_version()))
 342                         stability = impl.upstream_stability or model.testing
 343                         if highest_version is None or impl.version > highest_version.version:
 344                                 highest_version = impl
 345                         if stability == model.testing:
 346                                 testing_error = None
 347                                 if not impl.released:
 348                                         if not impl.local_path:
 349                                                 testing_error = "No release date on testing version"
 350                                 else:
 351                                         try:
 352                                                 released = time.strptime(impl.released, '%Y-%m-%d')
 353                                         except ValueError, ex:
 354                                                 testing_error = "Can't parse date"
 355                                         else:
 356                                                 ago = now - time.mktime(released)
 357                                                 if ago < 0:
 358                                                         testing_error = 'Release date is in the future!'
 359                                 if testing_error:
 360                                         raise SafeException("Version %s: %s (released %s)" % (impl.get_version(), testing_error, impl.released))
 361
 362                         # Old Windows versions use 32-bit integers to store versions. Newer versions use 64-bit ones, but in general
 363                         # keeping the numbers small is helpful.
 364                         for i in range(0, len(impl.version), 2):
 365                                 for x in impl.version[i]:
 366                                         if x >= (1 << 31):
 367                                                 raise SafeException("Version %s: component %s won't fit in a 32-bit signed integer" % (impl.get_version(), x))
 368
 369                 if highest_version and (highest_version.upstream_stability or model.testing) is model.testing:
 370                         print highlight('  Highest version (%s) is still "testing"' % highest_version.get_version(), 'YELLOW')
 371
 372                 for homepage in feed_obj.get_metadata(namespaces.XMLNS_IFACE, 'homepage'):
 373                         check_exists(homepage.content)
 374
 375                 for icon in feed_obj.get_metadata(namespaces.XMLNS_IFACE, 'icon'):
 376                         check_exists(icon.getAttribute('href'))
 377
 378         except (urllib2.HTTPError, httplib.BadStatusLine, socket.error, ftplib.error_perm), ex:
 379                 err_msg = str(ex).strip() or str(type(ex))
 380                 error_new_line('  ' + err_msg)
 381                 n_errors += 1
 382                 if options.verbose: traceback.print_exc()
 383         except SafeException, ex:
 384                 if options.verbose: raise
 385                 error_new_line('  ' + str(ex))
 386                 n_errors += 1
 387
 388 if n_errors == 0:
 389         print "OK"
 390 else:
 391         print "\nERRORS FOUND:", n_errors
 392         sys.exit(1)