pkg2zero

   1 #!/usr/bin/env python
   2 # Copyright (C) 2009, Thomas Leonard
   3 # See the COPYING file for details, or visit http://0install.net.
   4
   5 import sys, time
   6 from optparse import OptionParser
   7 import tempfile, shutil, os
   8 from xml.dom import minidom
   9 import subprocess
  10
  11 from zeroinstall.injector import model, qdom, distro
  12
  13 from support import read_child, add_node, DebMappings
  14
  15 manifest_algorithm = 'sha1new'
  16
  17 deb_category_to_freedesktop = {
  18         'devel' : 'Development',
  19         'web' : 'Network',
  20         'graphics' : 'Graphics',
  21         'games' : 'Game',
  22 }
  23
  24 valid_categories = [
  25         'AudioVideo',
  26         'Audio',
  27         'Video',
  28         'Development',
  29         'Education',
  30         'Game',
  31         'Graphics',
  32         'Network',
  33         'Office',
  34         'Settings',
  35         'System',
  36         'Utility',
  37 ]
  38
  39 # Parse command-line arguments
  40
  41 parser = OptionParser('usage: %prog [options] http://.../package.deb [target-feed.xml]\n'
  42                       '       %prog [options] package-name [target-feed.xml]\n'
  43                       'Publish a Debian package in a Zero Install feed.\n'
  44                       "target-feed.xml is created if it doesn't already exist.")
  45 parser.add_option("-p", "--packages-file", help="package index file")
  46 parser.add_option("-m", "--mirror", help="location of packages [http://ftp.debian.org/debian]")
  47 parser.add_option("-k", "--key", help="key to use for signing")
  48 (options, args) = parser.parse_args()
  49
  50 if len(args) < 1 or len(args) > 2:
  51         parser.print_help()
  52         sys.exit(1)
  53
  54 packages_base_url = (options.mirror or 'http://ftp.debian.org/debian') + '/'
  55
  56 pkg_sha1 = None
  57 pkg_sha2 = None
  58
  59 scheme = args[0].split(':', 1)[0]
  60 if scheme in ('http', 'https', 'ftp'):
  61         pkg_url = args[0]
  62 else:
  63         packages_file = options.packages_file or 'Packages'
  64         if not os.path.isfile(packages_file):
  65                 print >>sys.stderr, ("File '%s' not found (use -p to give its location).\n"
  66                         "Either download one (e.g. ftp://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.bz2),\n"
  67                         "or specify the full URL of the .deb package to use.") % packages_file
  68                 sys.exit(1)
  69         if packages_file.endswith('.bz2'):
  70                 import bz2
  71                 opener = bz2.BZ2File
  72         else:
  73                 opener = file
  74         pkg_name = args[0]
  75         pkg_data = "\n" + opener(packages_file).read()
  76         try:
  77                 i = pkg_data.index('\nPackage: %s\n' % pkg_name)
  78         except ValueError:
  79                 raise Exception("Package '%s' not found in Packages file '%s'." % (pkg_name, packages_file))
  80         j = pkg_data.find('\n\n', i)
  81         if j == -1:
  82                 pkg_info = pkg_data[i:]
  83         else:
  84                 pkg_info = pkg_data[i:j]
  85         filename = None
  86         for line in pkg_info.split('\n'):
  87                 if ':' in line and not line.startswith(' '):
  88                         key, value = line.split(':', 1)
  89                         if key == 'Filename':
  90                                 filename = value.strip()
  91                         elif key == 'SHA1':
  92                                 pkg_sha1 = value.strip()
  93                         elif key == 'SHA256':
  94                                 pkg_sha2 = value.strip()
  95         if filename is None:
  96                 raise Exception('Filename: field not found in package data:\n' + pkg_info)
  97         pkg_url = packages_base_url + filename
  98
  99 # Download .deb package, if required
 100
 101 deb_file = os.path.abspath(pkg_url.rsplit('/', 1)[1])
 102 if not os.path.exists(deb_file):
 103         print >>sys.stderr, "File '%s' not found, so downloading from %s..." % (deb_file, pkg_url)
 104         subprocess.check_call(['wget', pkg_url])
 105
 106 # Check digest, if known
 107
 108 if pkg_sha1:
 109         import hashlib
 110         m = hashlib.new('sha1')
 111         m.update(file(deb_file).read())
 112         actual = m.hexdigest()
 113         if actual != pkg_sha1:
 114                 raise Exception("Incorrect digest on .deb file! Was " + actual + ", but expected " + pkg_sha1)
 115         else:
 116                 print >>sys.stderr, "Package's digest matches value in Packages file (" + actual + "). Good."
 117 elif pkg_sha2:
 118         import hashlib
 119         m = hashlib.new('sha256')
 120         m.update(file(deb_file).read())
 121         actual = m.hexdigest()
 122         if actual != pkg_sha2:
 123                 raise Exception("Incorrect digest on .deb file! Was " + actual + ", but expected " + pkg_sha2)
 124         else:
 125                 print >>sys.stderr, "Package's digest matches value in repodata (" + actual + "). Good."
 126 else:
 127         print >>sys.stderr, "Note: no SHA1 digest known for this package, so not checking..."
 128
 129 # Load dependency mappings
 130 deb_mappings = DebMappings()
 131
 132 # Extract meta-data from .deb
 133
 134 details = read_child(['dpkg-deb', '--info', deb_file])
 135
 136 description_and_summary = details.split('\n Description: ')[1].split('\n')
 137 summary = description_and_summary[0]
 138 description = ''
 139 for x in description_and_summary[1:]:
 140         if not x: continue
 141         assert x[0] == ' '
 142         x = x[1:]
 143         if x[0] != ' ':
 144                 break
 145         if x == ' .':
 146                 description += '\n'
 147         else:
 148                 description += x[1:].replace('.  ', '. ') + '\n'
 149 description = description.strip()
 150
 151 pkg_name = '(unknown)'
 152 pkg_version = None
 153 pkg_arch = None
 154 category = None
 155 requires = []
 156 for line in details.split('\n'):
 157         if not line: continue
 158         assert line.startswith(' ')
 159         line = line[1:]
 160         if ':' in line:
 161                 key, value = line.split(':', 1)
 162                 value = value.strip()
 163                 if key == 'Section':
 164                         category = deb_category_to_freedesktop.get(value)
 165                         if not category:
 166                                 if value != 'libs':
 167                                         print >>sys.stderr, "Warning: no mapping for Debian category '%s'" % value
 168                 elif key == 'Package':
 169                         pkg_name = value
 170                 elif key == 'Version':
 171                         value = value.replace('cvs', '')
 172                         value = value.replace('svn', '')
 173                         pkg_version = distro.try_cleanup_distro_version(value)
 174                 elif key == 'Architecture':
 175                         if '-' in value:
 176                             arch, value = value.split('-', 1)
 177                         else:
 178                             arch = 'linux'
 179                         if value == 'amd64':
 180                                 value = 'x86_64'
 181                         elif value == 'all':
 182                                 value = '*'
 183                         pkg_arch = arch.capitalize() + '-' + value
 184                 elif key == 'Depends':
 185                         for x in value.split(','):
 186                                 req = deb_mappings.process(x)
 187                                 if req:
 188                                         requires.append(req)
 189
 190 # Unpack package, find binaries and .desktop files, and add to cache
 191
 192 possible_mains = []
 193 tmp = tempfile.mkdtemp(prefix = 'pkg2zero-')
 194 try:
 195         files = read_child(['dpkg-deb', '-X', deb_file, tmp])
 196
 197         for f in files.split('\n'):
 198                 full = os.path.join(tmp, f)
 199                 if f.endswith('.desktop'):
 200                         for line in file(full):
 201                                 if line.startswith('Categories'):
 202                                         for cat in line.split('=', 1)[1].split(';'):
 203                                                 cat = cat.strip()
 204                                                 if cat in valid_categories:
 205                                                         category = cat
 206                                                         break
 207                 elif f.startswith('./usr/bin/') or f.startswith('./usr/games/'):
 208                         if os.path.isfile(full):
 209                                 possible_mains.append(f[2:])
 210
 211         manifest = read_child(['0store', 'manifest', tmp, manifest_algorithm])
 212         digest = manifest.rsplit('\n', 2)[1]
 213         subprocess.check_call(['0store', 'add', digest, tmp])
 214 finally:
 215         shutil.rmtree(tmp)
 216
 217 if possible_mains:
 218         possible_mains = sorted(possible_mains, key = len)
 219         pkg_main = possible_mains[0]
 220         if len(possible_mains) > 1:
 221                 print "Warning: several possible main binaries found:"
 222                 print "- " + pkg_main + " (I chose this one)"
 223                 for x in possible_mains[1:]:
 224                         print "- " + x
 225 else:
 226         pkg_main = None
 227
 228 # Make sure we haven't added this version already...
 229
 230 if len(args) > 1:
 231         target_feed_file = args[1]
 232 else:
 233         target_feed_file = pkg_name + '.xml'
 234
 235 feed_uri = None
 236 if os.path.isfile(target_feed_file):
 237         dummy_dist = distro.Distribution()
 238         dom = qdom.parse(file(target_feed_file))
 239         old_target_feed = model.ZeroInstallFeed(dom, local_path = target_feed_file, distro = dummy_dist)
 240         existing_impl = old_target_feed.implementations.get(digest)
 241         if existing_impl:
 242                 print >>sys.stderr, ("Feed '%s' already contains an implementation with this digest!\n%s" % (target_feed_file, existing_impl))
 243                 sys.exit(1)
 244 else:
 245         # No target, so need to pick a URI
 246         feed_uri = deb_mappings.lookup(pkg_name)
 247         if feed_uri is None:
 248                 suggestion = deb_mappings.get_suggestion(pkg_name)
 249                 uri = raw_input('Enter the URI for this feed [%s]: ' % suggestion).strip()
 250                 if not uri:
 251                         uri = suggestion
 252                 assert uri.startswith('http://') or uri.startswith('https://') or uri.startswith('ftp://'), uri
 253                 feed_uri = uri
 254                 deb_mappings.add_mapping(pkg_name, uri)
 255
 256 # Create a local feed with just the new version...
 257
 258 template = '''<interface xmlns="http://zero-install.sourceforge.net/2004/injector/interface">
 259 </interface>'''
 260 doc = minidom.parseString(template)
 261 root = doc.documentElement
 262
 263 add_node(root, 'name', pkg_name)
 264 add_node(root, 'summary', summary)
 265 add_node(root, 'description', description)
 266 feed_for = add_node(root, 'feed-for', '')
 267 if feed_uri:
 268         feed_for.setAttribute('interface', feed_uri)
 269 if category:
 270         add_node(root, 'category', category)
 271
 272 package = add_node(root, 'package-implementation', '')
 273 package.setAttribute('package', pkg_name)
 274
 275 group = add_node(root, 'group', '')
 276 if pkg_arch:
 277         group.setAttribute('arch', pkg_arch)
 278 else:
 279         print >>sys.stderr, "No Architecture: field in .deb."
 280
 281 for req in requires:
 282         req_element = add_node(group, 'requires', before = '\n    ', after = '')
 283         req_element.setAttribute('interface', req.interface)
 284         binding = add_node(req_element, 'environment', before = '\n      ', after = '\n    ')
 285         binding.setAttribute('name', 'LD_LIBRARY_PATH')
 286         binding.setAttribute('insert', 'usr/lib')
 287
 288 if pkg_main:
 289         group.setAttribute('main', pkg_main)
 290         package.setAttribute('main', '/' + pkg_main)
 291
 292 impl = add_node(group, 'implementation', before = '\n    ', after = '\n  ')
 293 impl.setAttribute('id', digest)
 294 impl.setAttribute('version', pkg_version)
 295 impl.setAttribute('released', time.strftime('%Y-%m-%d'))
 296
 297 archive = add_node(impl, 'archive', before = '\n      ', after = '\n    ')
 298 archive.setAttribute('href', pkg_url)
 299 archive.setAttribute('size', str(os.path.getsize(deb_file)))
 300
 301 # Add our new version to the main feed...
 302
 303 output_stream = tempfile.NamedTemporaryFile(prefix = 'pkg2zero-')
 304 try:
 305         output_stream.write("<?xml version='1.0'?>\n")
 306         root.writexml(output_stream)
 307         output_stream.write('\n')
 308         output_stream.flush()
 309
 310         publishing_options = []
 311         if options.key:
 312                 # Note: 0publish < 0.16 requires the --xmlsign option too
 313                 publishing_options += ['--xmlsign', '--key', options.key]
 314         subprocess.check_call([os.environ['PUBLISH_COMMAND']] + publishing_options + ['--local', output_stream.name, target_feed_file])
 315         print "Added version %s to %s" % (pkg_version, target_feed_file)
 316 finally:
 317         output_stream.close()