rpm2zero

   1 #!/usr/bin/env python
   2 # Copyright (C) 2008, Thomas Leonard
   3 # Copyright (C) 2008, Anders F Bjorklund
   4 # See the COPYING file for details, or visit http://0install.net.
   5
   6 import sys, time
   7 from optparse import OptionParser
   8 import tempfile, shutil, os
   9 from xml.dom import minidom
  10 import gzip
  11 import xml.etree.ElementTree as ET # Python 2.5
  12 import subprocess
  13
  14 from zeroinstall.injector import model, qdom, distro
  15
  16 from support import read_child, add_node, RPMMappings
  17
  18 manifest_algorithm = 'sha1new'
  19
  20 rpm_group_to_freedesktop = {
  21         'Development/Libraries' : 'Development',
  22 }
  23
  24 valid_categories = [
  25         'AudioVideo',
  26         'Audio',
  27         'Video',
  28         'Development',
  29         'Education',
  30         'Game',
  31         'Graphics',
  32         'Network',
  33         'Office',
  34         'Settings',
  35         'System',
  36         'Utility',
  37 ]
  38
  39 # Parse command-line arguments
  40
  41 parser = OptionParser('usage: %prog [options] http://.../package.rpm [target-feed.xml]\n'
  42                       '       %prog [options] package-name [target-feed.xml]\n'
  43                       'Publish an RPM package in a Zero Install feed.\n'
  44                       "target-feed.xml is created if it doesn't already exist.")
  45 parser.add_option("-r", "--repomd-file", help="repository metadata file")
  46 parser.add_option("-m", "--mirror", help="location of packages [http://mirror.centos.org]")
  47 parser.add_option("-p", "--path", help="location of packages [centos/5/os/i386]")
  48 parser.add_option("-k", "--key", help="key to use for signing")
  49 (options, args) = parser.parse_args()
  50
  51 if len(args) < 1 or len(args) > 2:
  52         parser.print_help()
  53         sys.exit(1)
  54
  55 packages_base_url = (options.mirror or 'http://mirror.centos.org') + '/'
  56 packages_base_dir = (options.path or 'centos/5/os/i386') + '/'
  57
  58 pkg_data = None
  59 pkg_sha1 = None
  60
  61 scheme = args[0].split(':', 1)[0]
  62 if scheme in ('http', 'https', 'ftp'):
  63         pkg_url = args[0]
  64 else:
  65         repomd_file = options.repomd_file or 'repodata/repomd.xml'
  66         if not os.path.isfile(repomd_file):
  67                 print >>sys.stderr, ("File '%s' not found (use -r to give its location).\n"
  68                         "Either download one (e.g. http://mirror.centos.org/centos/5/os/i386/repodata/repomd.xml),\n"
  69                         "or specify the full URL of the .rpm package to use.") % repomd_file
  70                 sys.exit(1)
  71         pkg_name = args[0]
  72         primary_file = None
  73         repomd = minidom.parse(repomd_file)
  74         for data in repomd.getElementsByTagName("data"):
  75             if data.attributes["type"].nodeValue == "primary":
  76                 for node in data.getElementsByTagName("location"):
  77                     primary_file = node.attributes["href"].nodeValue
  78         location = None
  79         primary = ET.parse(gzip.open(primary_file))
  80         NS = "http://linux.duke.edu/metadata/common"
  81         metadata = primary.getroot()
  82         for package in metadata.findall("{%s}package" % NS):
  83                 if package.find("{%s}name" % NS).text == pkg_name:
  84                     pkg_data = package
  85                     location = pkg_data.find("{%s}location" % NS).get("href")
  86                     break
  87         if pkg_data is None:
  88                 raise Exception("Package '%s' not found in repodata." % pkg_name)
  89         if pkg_data.find("{%s}checksum" % NS).get("type") == "sha":
  90                 pkg_sha1 = pkg_data.find("{%s}checksum" % NS).text
  91         if location is None:
  92                 raise Exception('location tag not found in primary metadata:\n' + primary_file)
  93         pkg_url = packages_base_url + packages_base_dir + location
  94
  95 # Download .rpm package, if required
  96
  97 rpm_file = os.path.abspath(pkg_url.rsplit('/', 1)[1])
  98 if not os.path.exists(rpm_file):
  99         print >>sys.stderr, "File '%s' not found, so downloading from %s..." % (rpm_file, pkg_url)
 100         subprocess.check_call(['wget', pkg_url])
 101
 102 # Check digest, if known
 103
 104 if pkg_sha1:
 105         import hashlib
 106         m = hashlib.new('sha1')
 107         m.update(file(rpm_file).read())
 108         actual = m.hexdigest()
 109         if actual != pkg_sha1:
 110                 raise Exception("Incorrect digest on .rpm file! Was " + actual + ", but expected " + pkg_sha1)
 111         else:
 112                 print >>sys.stderr, "Package's digest matches value in repodata (" + actual + "). Good."
 113 else:
 114         print >>sys.stderr, "Note: no SHA1 digest known for this package, so not checking..."
 115
 116 # Load dependency mappings
 117 rpm_mappings = RPMMappings()
 118
 119 # Extract meta-data from .rpm
 120
 121 query_format = '%{SUMMARY}\\a%{DESCRIPTION}\\a%{NAME}\\a%{VERSION}\\a%{ARCH}\\a%{GROUP}\\a[%{REQUIRES}\\n]'
 122 headers = read_child(['rpm', '--qf', query_format, '-qp', rpm_file]).split('\a')
 123
 124 summary = headers[0].strip()
 125 description = headers[1].strip()
 126
 127 pkg_name = headers[2]
 128 value = headers[3]
 129 value = value.replace('cvs', '')
 130 value = value.replace('svn', '')
 131 value = distro.try_cleanup_distro_version(value)
 132 pkg_version = value
 133 value = headers[4]
 134 if value == 'amd64':
 135     value = 'x86_64'
 136 if value == 'noarch':
 137     value = '*'
 138 pkg_arch = 'Linux-' + value
 139 category = None
 140 value = headers[5].strip()
 141 category = rpm_group_to_freedesktop.get(value)
 142 if not category:
 143     print >>sys.stderr, "Warning: no mapping for RPM group '%s'" % value
 144 requires = []
 145 value = headers[6].strip()
 146 for x in value.split('\n'):
 147     if x.startswith('rpmlib'):
 148         continue
 149     req = rpm_mappings.process(x)
 150     if req:
 151         requires.append(req)
 152
 153 # Unpack package, find binaries and .desktop files, and add to cache
 154
 155 possible_mains = []
 156 tmp = tempfile.mkdtemp(prefix = 'rpm2zero-')
 157 try:
 158         files = read_child(['rpm', '-qlp', rpm_file])
 159         p1 = subprocess.Popen(['rpm2cpio', rpm_file], stdout=subprocess.PIPE)
 160         IGNORE = open('/dev/null', 'w')
 161         p2 = subprocess.Popen(['cpio', '-dim'], stdin=p1.stdout, stderr=IGNORE, cwd=tmp)
 162         p2.communicate()
 163
 164         for f in files.split('\n'):
 165                 full = os.path.join(tmp, f[1:])
 166                 if f.endswith('.desktop'):
 167                         for line in file(full):
 168                                 if line.startswith('Categories'):
 169                                         for cat in line.split('=', 1)[1].split(';'):
 170                                                 cat = cat.strip()
 171                                                 if cat in valid_categories:
 172                                                         category = cat
 173                                                         break
 174                 elif f.startswith('/usr/bin/') or f.startswith('/usr/games/'):
 175                         if os.path.isfile(full):
 176                                 possible_mains.append(f[1:])
 177
 178         manifest = read_child(['0store', 'manifest', tmp, manifest_algorithm])
 179         digest = manifest.rsplit('\n', 2)[1]
 180         subprocess.check_call(['0store', 'add', digest, tmp])
 181 finally:
 182         shutil.rmtree(tmp)
 183
 184 if possible_mains:
 185         possible_mains = sorted(possible_mains, key = len)
 186         pkg_main = possible_mains[0]
 187         if len(possible_mains) > 1:
 188                 print "Warning: several possible main binaries found:"
 189                 print "- " + pkg_main + " (I chose this one)"
 190                 for x in possible_mains[1:]:
 191                         print "- " + x
 192 else:
 193         pkg_main = None
 194
 195 # Make sure we haven't added this version already...
 196
 197 if len(args) > 1:
 198         target_feed_file = args[1]
 199 else:
 200         target_feed_file = pkg_name + '.xml'
 201
 202 feed_uri = None
 203 if os.path.isfile(target_feed_file):
 204         dummy_dist = distro.Distribution()
 205         dom = qdom.parse(file(target_feed_file))
 206         old_target_feed = model.ZeroInstallFeed(dom, local_path = target_feed_file, distro = dummy_dist)
 207         existing_impl = old_target_feed.implementations.get(digest)
 208         if existing_impl:
 209                 print >>sys.stderr, ("Feed '%s' already contains an implementation with this digest!\n%s" % (target_feed_file, existing_impl))
 210                 sys.exit(1)
 211 else:
 212         # No target, so need to pick a URI
 213         feed_uri = rpm_mappings.lookup(pkg_name)
 214         if feed_uri is None:
 215                 suggestion = rpm_mappings.get_suggestion(pkg_name)
 216                 uri = raw_input('Enter the URI for this feed [%s]: ' % suggestion).strip()
 217                 if not uri:
 218                         uri = suggestion
 219                 assert uri.startswith('http://') or uri.startswith('https://') or uri.startswith('ftp://'), uri
 220                 feed_uri = uri
 221                 rpm_mappings.add_mapping(pkg_name, uri)
 222
 223 # Create a local feed with just the new version...
 224
 225 template = '''<interface xmlns="http://zero-install.sourceforge.net/2004/injector/interface">
 226 </interface>'''
 227 doc = minidom.parseString(template)
 228 root = doc.documentElement
 229
 230 add_node(root, 'name', pkg_name)
 231 add_node(root, 'summary', summary)
 232 add_node(root, 'description', description)
 233 feed_for = add_node(root, 'feed-for', '')
 234 if feed_uri:
 235         feed_for.setAttribute('interface', feed_uri)
 236 if category:
 237         add_node(root, 'category', category)
 238
 239 package = add_node(root, 'package-implementation', '')
 240 package.setAttribute('package', pkg_name)
 241
 242 group = add_node(root, 'group', '')
 243 group.setAttribute('arch', pkg_arch)
 244
 245 for req in requires:
 246         req_element = add_node(group, 'requires', before = '\n    ', after = '')
 247         req_element.setAttribute('interface', req.interface)
 248         binding = add_node(req_element, 'environment', before = '\n      ', after = '\n    ')
 249         binding.setAttribute('name', 'LD_LIBRARY_PATH')
 250         binding.setAttribute('insert', 'usr/lib')
 251
 252 if pkg_main:
 253         group.setAttribute('main', pkg_main)
 254         package.setAttribute('main', '/' + pkg_main)
 255
 256 impl = add_node(group, 'implementation', before = '\n    ', after = '\n  ')
 257 impl.setAttribute('id', digest)
 258 impl.setAttribute('version', pkg_version)
 259 impl.setAttribute('released', time.strftime('%Y-%m-%d'))
 260
 261 archive = add_node(impl, 'archive', before = '\n      ', after = '\n    ')
 262 archive.setAttribute('href', pkg_url)
 263 archive.setAttribute('size', str(os.path.getsize(rpm_file)))
 264
 265 # Add our new version to the main feed...
 266
 267 output_stream = tempfile.NamedTemporaryFile(prefix = 'rpm2zero-')
 268 try:
 269         output_stream.write("<?xml version='1.0'?>\n")
 270         root.writexml(output_stream)
 271         output_stream.write('\n')
 272         output_stream.flush()
 273
 274         publishing_options = []
 275         if options.key:
 276                 # Note: 0publish < 0.16 requires the --xmlsign option too
 277                 publishing_options += ['--xmlsign', '--key', options.key]
 278         subprocess.check_call([os.environ['PUBLISH_COMMAND']] + publishing_options + ['--local', output_stream.name, target_feed_file])
 279         print "Added version %s to %s" % (pkg_version, target_feed_file)
 280 finally:
 281         output_stream.close()