feedlint

   1 #!/usr/bin/env python
   2
   3 from optparse import OptionParser
   4 import sys, shutil, tempfile, urlparse
   5 import socket
   6 import urllib2, os, httplib
   7 import ftplib
   8 import logging, time, traceback
   9 from logging import info
  10
  11 from zeroinstall import SafeException
  12 from zeroinstall.injector import model, gpg, reader, namespaces
  13
  14 from display import checking, result, error, highlight, error_new_line
  15
  16 now = time.time()
  17
  18 version = '0.4'
  19
  20 WEEK = 60 * 60 * 24 * 7
  21
  22 def host(address):
  23         if hasattr(address, 'hostname'):
  24                 return address.hostname
  25         else:
  26                 return address[1].split(':', 1)[0]
  27
  28 def port(address):
  29         if hasattr(address, 'port'):
  30                 return address.port
  31         else:
  32                 port = address[1].split(':', 1)[1:]
  33                 if port:
  34                         return int(port[0])
  35                 else:
  36                         return None
  37
  38 assert port(('http', 'foo:81')) == 81
  39 assert port(urlparse.urlparse('http://foo:81')) == 81
  40
  41 parser = OptionParser(usage="usage: %prog [options] feed.xml")
  42 parser.add_option("-d", "--dependencies", help="also check feeds for dependencies", action='store_true')
  43 parser.add_option("-s", "--skip-archives", help="don't check the archives are OK", action='store_true')
  44 parser.add_option("-v", "--verbose", help="more verbose output", action='count')
  45 parser.add_option("-V", "--version", help="display version information", action='store_true')
  46
  47 (options, args) = parser.parse_args()
  48
  49 if options.version:
  50         print "FeedLint (zero-install) " + version
  51         print "Copyright (C) 2007 Thomas Leonard"
  52         print "This program comes with ABSOLUTELY NO WARRANTY,"
  53         print "to the extent permitted by law."
  54         print "You may redistribute copies of this program"
  55         print "under the terms of the GNU General Public License."
  56         print "For more information about these matters, see the file named COPYING."
  57         sys.exit(0)
  58
  59 if options.verbose:
  60         logger = logging.getLogger()
  61         if options.verbose == 1:
  62                 logger.setLevel(logging.INFO)
  63         else:
  64                 logger.setLevel(logging.DEBUG)
  65
  66 if len(args) < 1:
  67         parser.print_help()
  68         sys.exit(1)
  69
  70 checked = set()
  71
  72 try:
  73         to_check = [model.canonical_iface_uri(a) for a in args]
  74 except SafeException, ex:
  75         if options.verbose: raise
  76         print >>sys.stderr, ex
  77         sys.exit(1)
  78
  79 def check_key(feed, fingerprint):
  80         for line in os.popen('gpg --with-colons --list-keys %s' % s.fingerprint):
  81                 if line.startswith('pub:'):
  82                         key_id = line.split(':')[4]
  83                         break
  84         else:
  85                 raise SafeException('Failed to find key with fingerprint %s on your keyring' % fingerprint)
  86
  87         key_url = urlparse.urljoin(feed, '%s.gpg' % key_id)
  88
  89         if key_url in checked:
  90                 info("(already checked key URL %s)", key_url)
  91         else:
  92                 checking("Checking key %s" % key_url)
  93                 urllib2.urlopen(key_url).read()
  94                 result('OK')
  95                 checked.add(key_url)
  96
  97 def get_http_size(url, ttl = 3):
  98         address = urlparse.urlparse(url)
  99
 100         if url.lower().startswith('http://'):
 101                 http = httplib.HTTPConnection(host(address), port(address) or 80)
 102         elif url.lower().startswith('https://'):
 103                 http = httplib.HTTPSConnection(host(address), port(address) or 443)
 104         else:
 105                 assert False, url
 106
 107         parts = url.split('/', 3)
 108         if len(parts) == 4:
 109                 path = parts[3]
 110         else:
 111                 path = ''
 112
 113         http.request('HEAD', '/' + path, headers = {'Host': host(address)})
 114         response = http.getresponse()
 115         try:
 116                 if response.status == 200:
 117                         return response.getheader('Content-Length')
 118                 elif response.status in (301, 302):
 119                         new_url_rel = response.getheader('Location') or response.getheader('URI')
 120                         new_url = urlparse.urljoin(url, new_url_rel)
 121                 else:
 122                         raise SafeException("HTTP error: got status code %s" % response.status)
 123         finally:
 124                 response.close()
 125
 126         if ttl:
 127                 result("Moved", 'YELLOW')
 128                 checking("Checking new URL %s" % new_url)
 129                 assert new_url
 130                 return get_http_size(new_url, ttl - 1)
 131         else:
 132                 raise SafeException('Too many redirections.')
 133
 134 def get_ftp_size(url):
 135         address = urlparse.urlparse(url)
 136         ftp = ftplib.FTP(host(address))
 137         try:
 138                 ftp.login()
 139                 ftp.voidcmd('TYPE I')
 140                 return ftp.size(url.split('/', 3)[3])
 141         finally:
 142                 ftp.close()
 143
 144 def get_size(url):
 145         scheme = urlparse.urlparse(url)[0].lower()
 146         if scheme.startswith('http') or scheme.startswith('https'):
 147                 return get_http_size(url)
 148         elif scheme.startswith('ftp'):
 149                 return get_ftp_size(url)
 150         else:
 151                 raise SafeException("Unknown scheme '%s' in '%s'" % (scheme, url))
 152
 153 def check_source(source):
 154         if hasattr(source, 'url'):
 155                 checking("Checking archive %s" % source.url)
 156                 actual_size = get_size(source.url)
 157                 if actual_size is None:
 158                         result("No Content-Length for archive; can't check", 'YELLOW')
 159                 else:
 160                         actual_size = int(actual_size)
 161                         expected_size = source.size + (source.start_offset or 0)
 162                         if actual_size != expected_size:
 163                                 error('Bad length')
 164                                 raise SafeException("Expected archive to have a size of %d, but server says it is %d" %
 165                                                 (expected_size, actual_size))
 166                         result('OK')
 167         elif hasattr(source, 'steps'):
 168                 for step in source.steps:
 169                         check_source(step)
 170
 171 existing_urls = set()
 172 def check_exists(url):
 173         if url in existing_urls: return         # Already checked
 174
 175         checking("Checking URL exists %s" % url)
 176         get_size(url)
 177         result('OK')
 178         existing_urls.add(url)
 179
 180 n_errors = 0
 181
 182 while to_check:
 183         feed = to_check.pop()
 184         if feed in checked:
 185                 info("Already checked feed %s", feed)
 186                 continue
 187
 188         checked.add(feed)
 189
 190         checking("Checking " + feed, indent = 0)
 191
 192         try:
 193                 iface = model.Interface(feed)
 194
 195                 if feed.startswith('/'):
 196                         reader.update(iface, feed, local = True)
 197                         result('Local')
 198                 else:
 199                         tmp = tempfile.TemporaryFile(prefix = 'feedlint-')
 200                         try:
 201                                 stream = urllib2.urlopen(feed)
 202                                 shutil.copyfileobj(stream, tmp)
 203                                 tmp.seek(0)
 204                                 start = tmp.read(5)
 205                                 if start == '<?xml':
 206                                         result('Fetched')
 207                                 elif start == '-----':
 208                                         result('Old sig', colour = 'YELLOW')
 209                                         error_new_line('  Feed has an old-style plain GPG signature. Use 0publish --xmlsign.',
 210                                                         colour = 'YELLOW')
 211                                 else:
 212                                         result('Fetched')
 213                                         error_new_line('  Unknown format. File starts "%s"' % start)
 214
 215                                 tmp.seek(0)
 216                                 data, sigs = gpg.check_stream(tmp)
 217
 218                                 for s in sigs:
 219                                         if isinstance(s, gpg.ValidSig):
 220                                                 check_key(feed, s.fingerprint)
 221                                         else:
 222                                                 raise SafeException("Can't check sig: %s" % s)
 223
 224                                 feed_tmp = tempfile.NamedTemporaryFile(prefix = 'feedlint-')
 225                                 try:
 226                                         shutil.copyfileobj(data, feed_tmp)
 227                                         feed_tmp.seek(0)
 228                                         reader.update(iface, feed_tmp.name)
 229
 230                                         for f in iface.feeds:
 231                                                 info("Will check feed %s", f.uri)
 232                                                 to_check.append(f.uri)
 233                                 finally:
 234                                         feed_tmp.close()
 235                         finally:
 236                                 tmp.close()
 237
 238                 for impl in iface.implementations.values():
 239                         if hasattr(impl, 'dependencies'):
 240                                 for r in impl.dependencies.values():
 241                                         if r.interface not in checked:
 242                                                 info("Will check dependency %s", r)
 243                                                 if options.dependencies:
 244                                                         to_check.append(r.interface)
 245                                                 else:
 246                                                         check_exists(r.interface)
 247                         if hasattr(impl, 'download_sources') and not options.skip_archives:
 248                                 for source in impl.download_sources:
 249                                         check_source(source)
 250                         stability = impl.upstream_stability or model.testing
 251                         if stability == model.testing:
 252                                 testing_error = None
 253                                 if not impl.released:
 254                                         testing_error = "No release data on testing version"
 255                                 else:
 256                                         try:
 257                                                 released = time.strptime(impl.released, '%Y-%m-%d')
 258                                         except ValueError, ex:
 259                                                 testing_error = "Can't parse date"
 260                                         else:
 261                                                 ago = now - time.mktime(released)
 262                                                 if ago < 0:
 263                                                         testing_error = 'Release data is in the future!'
 264                                                 elif ago > 1 * WEEK:
 265                                                         print highlight('  Version %s has been Testing for more than a week' % impl.get_version(),
 266                                                                         'YELLOW')
 267                                 if testing_error:
 268                                         raise SafeException("Version %s: %s (released %s)" % (impl.get_version(), testing_error, impl.released))
 269
 270                 for homepage in iface.get_metadata(namespaces.XMLNS_IFACE, 'homepage'):
 271                         check_exists(homepage.content)
 272
 273                 for icon in iface.get_metadata(namespaces.XMLNS_IFACE, 'icon'):
 274                         check_exists(icon.getAttribute('href'))
 275
 276         except (urllib2.HTTPError, httplib.BadStatusLine, socket.error, ftplib.error_perm), ex:
 277                 err_msg = str(ex).strip() or str(type(ex))
 278                 error_new_line('  ' + err_msg)
 279                 n_errors += 1
 280                 if options.verbose: traceback.print_exc()
 281         except SafeException, ex:
 282                 if options.verbose: raise
 283                 error_new_line('  ' + str(ex))
 284                 n_errors += 1
 285
 286 if n_errors == 0:
 287         print "OK"
 288 else:
 289         print "\nERRORS FOUND:", n_errors
 290         sys.exit(1)