rpm2zero

   1 #!/usr/bin/env python
   2 # Copyright (C) 2008, Thomas Leonard
   3 # Copyright (C) 2008, Anders F Bjorklund
   4 # See the COPYING file for details, or visit http://0install.net.
   5
   6 import sys, time
   7 from optparse import OptionParser
   8 import tempfile, shutil, os
   9 from xml.dom import minidom
  10 import gzip
  11 try:
  12     import xml.etree.cElementTree as ET # Python 2.5
  13 except ImportError:
  14     try:
  15         import xml.etree.ElementTree as ET
  16     except ImportError:
  17         try:
  18             import cElementTree as ET # http://effbot.org
  19         except ImportError:
  20             import elementtree.ElementTree as ET
  21
  22 import subprocess
  23 try:
  24     from subprocess import check_call
  25 except ImportError:
  26     def check_call(*popenargs, **kwargs):
  27         rc = subprocess.call(*popenargs, **kwargs)
  28         if rc != 0: raise(OSError, str(rc))
  29
  30 from zeroinstall.injector import model, qdom, distro
  31
  32 from support import read_child, add_node, RPMMappings
  33
  34 manifest_algorithm = 'sha1new'
  35
  36 rpm_group_to_freedesktop = {
  37         'Development/Libraries' : 'Development',
  38 }
  39
  40 valid_categories = [
  41         'AudioVideo',
  42         'Audio',
  43         'Video',
  44         'Development',
  45         'Education',
  46         'Game',
  47         'Graphics',
  48         'Network',
  49         'Office',
  50         'Settings',
  51         'System',
  52         'Utility',
  53 ]
  54
  55 # Parse command-line arguments
  56
  57 parser = OptionParser('usage: %prog [options] http://.../package.rpm [target-feed.xml]\n'
  58                       '       %prog [options] package-name [target-feed.xml]\n'
  59                       'Publish an RPM package in a Zero Install feed.\n'
  60                       "target-feed.xml is created if it doesn't already exist.")
  61 parser.add_option("-r", "--repomd-file", help="repository metadata file")
  62 parser.add_option("-m", "--mirror", help="location of packages [http://mirror.centos.org]")
  63 parser.add_option("-p", "--path", help="location of packages [centos/5/os/i386]")
  64 parser.add_option("-k", "--key", help="key to use for signing")
  65 (options, args) = parser.parse_args()
  66
  67 if len(args) < 1 or len(args) > 2:
  68         parser.print_help()
  69         sys.exit(1)
  70
  71 packages_base_url = (options.mirror or 'http://mirror.centos.org') + '/'
  72 packages_base_dir = (options.path or 'centos/5/os/i386') + '/'
  73
  74 pkg_data = None
  75 pkg_sha1 = None
  76
  77 scheme = args[0].split(':', 1)[0]
  78 if scheme in ('http', 'https', 'ftp'):
  79         pkg_url = args[0]
  80 else:
  81         repomd_file = options.repomd_file or 'repodata/repomd.xml'
  82         if not os.path.isfile(repomd_file):
  83                 print >>sys.stderr, ("File '%s' not found (use -r to give its location).\n"
  84                         "Either download one (e.g. http://mirror.centos.org/centos/5/os/i386/repodata/repomd.xml),\n"
  85                         "or specify the full URL of the .rpm package to use.") % repomd_file
  86                 sys.exit(1)
  87         pkg_name = args[0]
  88         primary_file = None
  89         repomd = minidom.parse(repomd_file)
  90         for data in repomd.getElementsByTagName("data"):
  91             if data.attributes["type"].nodeValue == "primary":
  92                 for node in data.getElementsByTagName("location"):
  93                     primary_file = node.attributes["href"].nodeValue
  94         location = None
  95         primary = ET.parse(gzip.open(primary_file))
  96         NS = "http://linux.duke.edu/metadata/common"
  97         metadata = primary.getroot()
  98         for package in metadata.findall("{%s}package" % NS):
  99                 if package.find("{%s}name" % NS).text == pkg_name:
 100                     pkg_data = package
 101                     location = pkg_data.find("{%s}location" % NS).get("href")
 102                     break
 103         if pkg_data is None:
 104                 raise Exception("Package '%s' not found in repodata." % pkg_name)
 105         if pkg_data.find("{%s}checksum" % NS).get("type") == "sha":
 106                 pkg_sha1 = pkg_data.find("{%s}checksum" % NS).text
 107         if location is None:
 108                 raise Exception('location tag not found in primary metadata:\n' + primary_file)
 109         pkg_url = packages_base_url + packages_base_dir + location
 110
 111 # Download .rpm package, if required
 112
 113 rpm_file = os.path.abspath(pkg_url.rsplit('/', 1)[1])
 114 if not os.path.exists(rpm_file):
 115         print >>sys.stderr, "File '%s' not found, so downloading from %s..." % (rpm_file, pkg_url)
 116         check_call(['wget', pkg_url])
 117
 118 # Check digest, if known
 119
 120 if pkg_sha1:
 121         try:
 122             import hashlib
 123             m = hashlib.new('sha1')
 124         except ImportError:
 125             import sha
 126             m = sha.new()
 127         m.update(file(rpm_file).read())
 128         actual = m.hexdigest()
 129         if actual != pkg_sha1:
 130                 raise Exception("Incorrect digest on .rpm file! Was " + actual + ", but expected " + pkg_sha1)
 131         else:
 132                 print >>sys.stderr, "Package's digest matches value in repodata (" + actual + "). Good."
 133 else:
 134         print >>sys.stderr, "Note: no SHA1 digest known for this package, so not checking..."
 135
 136 # Load dependency mappings
 137 rpm_mappings = RPMMappings()
 138
 139 # Extract meta-data from .rpm
 140
 141 query_format = '%{SUMMARY}\\a%{DESCRIPTION}\\a%{NAME}\\a%{VERSION}\\a%{OS}\\a%{ARCH}\\a%{GROUP}\\a%{LICENSE}\\a[%{REQUIRES}\\n]'
 142 headers = read_child(['rpm', '--qf', query_format, '-qp', rpm_file]).split('\a')
 143
 144 summary = headers[0].strip()
 145 description = headers[1].strip()
 146
 147 pkg_name = headers[2]
 148 value = headers[3]
 149 value = value.replace('cvs', '')
 150 value = value.replace('svn', '')
 151 value = distro.try_cleanup_distro_version(value)
 152 pkg_version = value
 153 value = headers[4]
 154 pkg_arch = value.capitalize()
 155 value = headers[5]
 156 if value == 'amd64':
 157     value = 'x86_64'
 158 if value == 'noarch':
 159     value = '*'
 160 pkg_arch += '-' + value
 161 category = None
 162 value = headers[6].strip()
 163 category = rpm_group_to_freedesktop.get(value)
 164 if not category:
 165     print >>sys.stderr, "Warning: no mapping for RPM group '%s'" % value
 166 requires = []
 167 value = headers[7].strip()
 168 license = value
 169 value = headers[8].strip()
 170 for x in value.split('\n'):
 171     if x.startswith('rpmlib'):
 172         continue
 173     req = rpm_mappings.process(x)
 174     if req:
 175         requires.append(req)
 176
 177 # Unpack package, find binaries and .desktop files, and add to cache
 178
 179 possible_mains = []
 180 tmp = tempfile.mkdtemp(prefix = 'rpm2zero-')
 181 try:
 182         files = read_child(['rpm', '-qlp', rpm_file])
 183         p1 = subprocess.Popen(['rpm2cpio', rpm_file], stdout=subprocess.PIPE)
 184         IGNORE = open('/dev/null', 'w')
 185         p2 = subprocess.Popen(['cpio', '-dim'], stdin=p1.stdout, stderr=IGNORE, cwd=tmp)
 186         p2.communicate()
 187
 188         for f in files.split('\n'):
 189                 full = os.path.join(tmp, f[1:])
 190                 if f.endswith('.desktop'):
 191                         for line in file(full):
 192                                 if line.startswith('Categories'):
 193                                         for cat in line.split('=', 1)[1].split(';'):
 194                                                 cat = cat.strip()
 195                                                 if cat in valid_categories:
 196                                                         category = cat
 197                                                         break
 198                 elif f.startswith('/usr/bin/') or f.startswith('/usr/games/'):
 199                         if os.path.isfile(full):
 200                                 possible_mains.append(f[1:])
 201
 202         manifest = read_child(['0store', 'manifest', tmp, manifest_algorithm])
 203         digest = manifest.rsplit('\n', 2)[1]
 204         check_call(['0store', 'add', digest, tmp])
 205 finally:
 206         shutil.rmtree(tmp)
 207
 208 if possible_mains:
 209         possible_mains = sorted(possible_mains, key = len)
 210         pkg_main = possible_mains[0]
 211         if len(possible_mains) > 1:
 212                 print "Warning: several possible main binaries found:"
 213                 print "- " + pkg_main + " (I chose this one)"
 214                 for x in possible_mains[1:]:
 215                         print "- " + x
 216 else:
 217         pkg_main = None
 218
 219 # Make sure we haven't added this version already...
 220
 221 if len(args) > 1:
 222         target_feed_file = args[1]
 223 else:
 224         target_feed_file = pkg_name + '.xml'
 225
 226 feed_uri = None
 227 if os.path.isfile(target_feed_file):
 228         dummy_dist = distro.Distribution()
 229         dom = qdom.parse(file(target_feed_file))
 230         old_target_feed = model.ZeroInstallFeed(dom, local_path = target_feed_file, distro = dummy_dist)
 231         existing_impl = old_target_feed.implementations.get(digest)
 232         if existing_impl:
 233                 print >>sys.stderr, ("Feed '%s' already contains an implementation with this digest!\n%s" % (target_feed_file, existing_impl))
 234                 sys.exit(1)
 235 else:
 236         # No target, so need to pick a URI
 237         feed_uri = rpm_mappings.lookup(pkg_name)
 238         if feed_uri is None:
 239                 suggestion = rpm_mappings.get_suggestion(pkg_name)
 240                 uri = raw_input('Enter the URI for this feed [%s]: ' % suggestion).strip()
 241                 if not uri:
 242                         uri = suggestion
 243                 assert uri.startswith('http://') or uri.startswith('https://') or uri.startswith('ftp://'), uri
 244                 feed_uri = uri
 245                 rpm_mappings.add_mapping(pkg_name, uri)
 246
 247 # Create a local feed with just the new version...
 248
 249 template = '''<interface xmlns="http://zero-install.sourceforge.net/2004/injector/interface">
 250 </interface>'''
 251 doc = minidom.parseString(template)
 252 root = doc.documentElement
 253
 254 add_node(root, 'name', pkg_name)
 255 add_node(root, 'summary', summary)
 256 add_node(root, 'description', description)
 257 feed_for = add_node(root, 'feed-for', '')
 258 if feed_uri:
 259         feed_for.setAttribute('interface', feed_uri)
 260 if category:
 261         add_node(root, 'category', category)
 262
 263 package = add_node(root, 'package-implementation', '')
 264 package.setAttribute('package', pkg_name)
 265
 266 group = add_node(root, 'group', '')
 267 group.setAttribute('arch', pkg_arch)
 268 group.setAttribute('license', license)
 269
 270 for req in requires:
 271         req_element = add_node(group, 'requires', before = '\n    ', after = '')
 272         req_element.setAttribute('interface', req.interface)
 273         binding = add_node(req_element, 'environment', before = '\n      ', after = '\n    ')
 274         binding.setAttribute('name', 'LD_LIBRARY_PATH')
 275         binding.setAttribute('insert', 'usr/lib')
 276
 277 if pkg_main:
 278         group.setAttribute('main', pkg_main)
 279         package.setAttribute('main', '/' + pkg_main)
 280
 281 impl = add_node(group, 'implementation', before = '\n    ', after = '\n  ')
 282 impl.setAttribute('id', digest)
 283 impl.setAttribute('version', pkg_version)
 284 impl.setAttribute('released', time.strftime('%Y-%m-%d'))
 285
 286 archive = add_node(impl, 'archive', before = '\n      ', after = '\n    ')
 287 archive.setAttribute('href', pkg_url)
 288 archive.setAttribute('size', str(os.path.getsize(rpm_file)))
 289
 290 # Add our new version to the main feed...
 291
 292 output_stream = tempfile.NamedTemporaryFile(prefix = 'rpm2zero-')
 293 try:
 294         output_stream.write("<?xml version='1.0'?>\n")
 295         root.writexml(output_stream)
 296         output_stream.write('\n')
 297         output_stream.flush()
 298
 299         publishing_options = []
 300         if options.key:
 301                 # Note: 0publish < 0.16 requires the --xmlsign option too
 302                 publishing_options += ['--xmlsign', '--key', options.key]
 303         check_call([os.environ['PUBLISH_COMMAND']] + publishing_options + ['--local', output_stream.name, target_feed_file])
 304         print "Added version %s to %s" % (pkg_version, target_feed_file)
 305 finally:
 306         output_stream.close()