rpm2zero

   1 #!/usr/bin/env python
   2 # Copyright (C) 2008, Thomas Leonard
   3 # Copyright (C) 2008, Anders F Bjorklund
   4 # See the COPYING file for details, or visit http://0install.net.
   5
   6 import sys, time
   7 from optparse import OptionParser
   8 import tempfile, shutil, os
   9 from xml.dom import minidom
  10 import gzip
  11 try:
  12     import xml.etree.cElementTree as ET # Python 2.5
  13 except ImportError:
  14     try:
  15         import xml.etree.ElementTree as ET
  16     except ImportError:
  17         try:
  18             import cElementTree as ET # http://effbot.org
  19         except ImportError:
  20             import elementtree.ElementTree as ET
  21
  22 import subprocess
  23 try:
  24     from subprocess import check_call
  25 except ImportError:
  26     def check_call(*popenargs, **kwargs):
  27         rc = subprocess.call(*popenargs, **kwargs)
  28         if rc != 0: raise OSError, rc
  29
  30 from zeroinstall.injector import model, qdom, distro
  31
  32 from support import read_child, add_node, RPMMappings
  33
  34 manifest_algorithm = 'sha1new'
  35
  36 rpm_group_to_freedesktop = {
  37         'Development/Libraries' : 'Development',
  38 }
  39
  40 valid_categories = [
  41         'AudioVideo',
  42         'Audio',
  43         'Video',
  44         'Development',
  45         'Education',
  46         'Game',
  47         'Graphics',
  48         'Network',
  49         'Office',
  50         'Settings',
  51         'System',
  52         'Utility',
  53 ]
  54
  55 # Parse command-line arguments
  56
  57 parser = OptionParser('usage: %prog [options] http://.../package.rpm [target-feed.xml]\n'
  58                       '       %prog [options] package-name [target-feed.xml]\n'
  59                       'Publish an RPM package in a Zero Install feed.\n'
  60                       "target-feed.xml is created if it doesn't already exist.")
  61 parser.add_option("-r", "--repomd-file", help="repository metadata file")
  62 parser.add_option("-m", "--mirror", help="location of packages [http://mirror.centos.org/centos]")
  63 parser.add_option("-p", "--path", help="location of packages [5/os/i386]")
  64 parser.add_option("-k", "--key", help="key to use for signing")
  65 (options, args) = parser.parse_args()
  66
  67 if len(args) < 1 or len(args) > 2:
  68         parser.print_help()
  69         sys.exit(1)
  70
  71 packages_base_url = (options.mirror or 'http://mirror.centos.org/centos') + '/'
  72 packages_base_dir = (options.path or '5/os/i386') + '/'
  73
  74 pkg_data = None
  75 pkg_sha1 = None
  76 pkg_sha2 = None
  77
  78 scheme = args[0].split(':', 1)[0]
  79 if scheme in ('http', 'https', 'ftp'):
  80         pkg_url = args[0]
  81 else:
  82         repomd_file = options.repomd_file or 'repodata/repomd.xml'
  83         if not os.path.isfile(repomd_file):
  84                 print >>sys.stderr, ("File '%s' not found (use -r to give its location).\n"
  85                         "Either download one (e.g. http://mirror.centos.org/centos/5/os/i386/repodata/repomd.xml),\n"
  86                         "or specify the full URL of the .rpm package to use.") % repomd_file
  87                 sys.exit(1)
  88         pkg_name = args[0]
  89         primary_file = None
  90         repomd = minidom.parse(repomd_file)
  91         for data in repomd.getElementsByTagName("data"):
  92             if data.attributes["type"].nodeValue == "primary":
  93                 for node in data.getElementsByTagName("location"):
  94                     primary_file = node.attributes["href"].nodeValue
  95         location = None
  96         primary = ET.parse(gzip.open(primary_file))
  97         NS = "http://linux.duke.edu/metadata/common"
  98         metadata = primary.getroot()
  99         for package in metadata.findall("{%s}package" % NS):
 100                 if package.find("{%s}name" % NS).text == pkg_name:
 101                     pkg_data = package
 102                     location = pkg_data.find("{%s}location" % NS).get("href")
 103                     break
 104         if pkg_data is None:
 105                 raise Exception("Package '%s' not found in repodata." % pkg_name)
 106         checksum = pkg_data.find("{%s}checksum" % NS)
 107         if checksum.get("type") == "sha":
 108                 pkg_sha1 = checksum.text
 109         if checksum.get("type") == "sha256":
 110                 pkg_sha2 = checksum.text
 111         if location is None:
 112                 raise Exception('location tag not found in primary metadata:\n' + primary_file)
 113         pkg_url = packages_base_url + packages_base_dir + location
 114
 115 # Download .rpm package, if required
 116
 117 rpm_file = os.path.abspath(pkg_url.rsplit('/', 1)[1])
 118 if not os.path.exists(rpm_file):
 119         print >>sys.stderr, "File '%s' not found, so downloading from %s..." % (rpm_file, pkg_url)
 120         check_call(['wget', pkg_url])
 121
 122 # Check digest, if known
 123
 124 if pkg_sha1:
 125         try:
 126             import hashlib
 127             m = hashlib.new('sha1')
 128         except ImportError:
 129             import sha
 130             m = sha.new()
 131         m.update(file(rpm_file).read())
 132         actual = m.hexdigest()
 133         if actual != pkg_sha1:
 134                 raise Exception("Incorrect digest on .rpm file! Was " + actual + ", but expected " + pkg_sha1)
 135         else:
 136                 print >>sys.stderr, "Package's digest matches value in repodata (" + actual + "). Good."
 137 elif pkg_sha2:
 138         import hashlib
 139         m = hashlib.new('sha256')
 140         m.update(file(rpm_file).read())
 141         actual = m.hexdigest()
 142         if actual != pkg_sha2:
 143                 raise Exception("Incorrect digest on .rpm file! Was " + actual + ", but expected " + pkg_sha2)
 144         else:
 145                 print >>sys.stderr, "Package's digest matches value in repodata (" + actual + "). Good."
 146 else:
 147         print >>sys.stderr, "Note: no SHA1 digest known for this package, so not checking..."
 148
 149 # Load dependency mappings
 150 rpm_mappings = RPMMappings()
 151
 152 # Extract meta-data from .rpm
 153
 154 query_format = '%{SUMMARY}\\a%{DESCRIPTION}\\a%{NAME}\\a%{VERSION}\\a%{OS}\\a%{ARCH}\\a%{URL}\\a%{GROUP}\\a%{LICENSE}\\a%{BUILDTIME}\\a[%{REQUIRES}\\n]'
 155 headers = read_child(['rpm', '--qf', query_format, '-qp', rpm_file]).split('\a')
 156
 157 summary = headers[0].strip()
 158 description = headers[1].strip()
 159
 160 pkg_name = headers[2]
 161 value = headers[3]
 162 value = value.replace('cvs', '')
 163 value = value.replace('svn', '')
 164 value = distro.try_cleanup_distro_version(value)
 165 pkg_version = value
 166 value = headers[4]
 167 pkg_arch = value.capitalize()
 168 value = headers[5]
 169 if value == 'amd64':
 170     value = 'x86_64'
 171 if value == 'noarch':
 172     value = '*'
 173 pkg_arch += '-' + value
 174 value = headers[6].strip()
 175 homepage = value
 176 category = None
 177 value = headers[7].strip()
 178 category = rpm_group_to_freedesktop.get(value)
 179 if not category:
 180     print >>sys.stderr, "Warning: no mapping for RPM group '%s'" % value
 181 requires = []
 182 value = headers[8].strip()
 183 license = value
 184 value = headers[9].strip()
 185 buildtime = long(value)
 186 value = headers[10].strip()
 187 for x in value.split('\n'):
 188     if x.startswith('rpmlib'):
 189         continue
 190     req = rpm_mappings.process(x)
 191     if req:
 192         requires.append(req)
 193
 194 # Unpack package, find binaries and .desktop files, and add to cache
 195
 196 possible_mains = []
 197 icondata = None
 198 tmp = tempfile.mkdtemp(prefix = 'rpm2zero-')
 199 try:
 200         files = read_child(['rpm', '-qlp', rpm_file])
 201         p1 = subprocess.Popen(['rpm2cpio', rpm_file], stdout=subprocess.PIPE)
 202         IGNORE = open('/dev/null', 'w')
 203         p2 = subprocess.Popen(['cpio', '-dim'], stdin=p1.stdout, stderr=IGNORE, cwd=tmp)
 204         p2.communicate()
 205
 206         icon = None
 207         images = {}
 208         for f in files.split('\n'):
 209                 full = os.path.join(tmp, f[1:])
 210                 if f.endswith('.desktop'):
 211                         for line in file(full):
 212                                 if line.startswith('Categories'):
 213                                         for cat in line.split('=', 1)[1].split(';'):
 214                                                 cat = cat.strip()
 215                                                 if cat in valid_categories:
 216                                                         category = cat
 217                                                         break
 218                                 elif line.startswith('Icon'):
 219                                         icon = line.split('=', 1)[1].strip()
 220                 elif f.startswith('/usr/bin/') or f.startswith('/usr/games/'):
 221                         if os.path.isfile(full):
 222                                 possible_mains.append(f[1:])
 223                 elif f.endswith('.png'):
 224                         images[f] = full
 225                         images[os.path.basename(f)] = full
 226                         # make sure to also map basename without the extension
 227                         images[os.path.splitext(os.path.basename(f))[0]] = full
 228
 229         icondata = None
 230         if icon in images:
 231                 print "Using %s for icon" % os.path.basename(images[icon])
 232                 icondata = file(images[icon]).read()
 233
 234         manifest = read_child(['0store', 'manifest', tmp, manifest_algorithm])
 235         digest = manifest.rsplit('\n', 2)[1]
 236         check_call(['0store', 'add', digest, tmp])
 237 finally:
 238         shutil.rmtree(tmp)
 239
 240 if possible_mains:
 241         possible_mains = sorted(possible_mains, key = len)
 242         pkg_main = possible_mains[0]
 243         if len(possible_mains) > 1:
 244                 print "Warning: several possible main binaries found:"
 245                 print "- " + pkg_main + " (I chose this one)"
 246                 for x in possible_mains[1:]:
 247                         print "- " + x
 248 else:
 249         pkg_main = None
 250
 251 # Make sure we haven't added this version already...
 252
 253 if len(args) > 1:
 254         target_feed_file = args[1]
 255         target_icon_file = args[1].replace('.xml', '.png')
 256 else:
 257         target_feed_file = pkg_name + '.xml'
 258         target_icon_file = pkg_name + '.png'
 259
 260 feed_uri = None
 261 icon_uri = None
 262 if os.path.isfile(target_feed_file):
 263         dummy_dist = distro.Distribution()
 264         dom = qdom.parse(file(target_feed_file))
 265         old_target_feed = model.ZeroInstallFeed(dom, local_path = target_feed_file, distro = dummy_dist)
 266         existing_impl = old_target_feed.implementations.get(digest)
 267         if existing_impl:
 268                 print >>sys.stderr, ("Feed '%s' already contains an implementation with this digest!\n%s" % (target_feed_file, existing_impl))
 269                 sys.exit(1)
 270 else:
 271         # No target, so need to pick a URI
 272         feed_uri = rpm_mappings.lookup(pkg_name)
 273         if feed_uri is None:
 274                 suggestion = rpm_mappings.get_suggestion(pkg_name)
 275                 uri = raw_input('Enter the URI for this feed [%s]: ' % suggestion).strip()
 276                 if not uri:
 277                         uri = suggestion
 278                 assert uri.startswith('http://') or uri.startswith('https://') or uri.startswith('ftp://'), uri
 279                 feed_uri = uri
 280                 rpm_mappings.add_mapping(pkg_name, uri)
 281
 282         if icondata and not os.path.isfile(target_icon_file):
 283                 file = open(target_icon_file, 'wb')
 284                 file.write(icondata)
 285                 file.close()
 286                 icon_uri = target_icon_file
 287
 288 # Create a local feed with just the new version...
 289
 290 template = '''<interface xmlns="http://zero-install.sourceforge.net/2004/injector/interface">
 291 </interface>'''
 292 doc = minidom.parseString(template)
 293 root = doc.documentElement
 294
 295 add_node(root, 'name', pkg_name)
 296 add_node(root, 'summary', summary)
 297 add_node(root, 'description', description)
 298 feed_for = add_node(root, 'feed-for', '')
 299 if feed_uri:
 300         feed_for.setAttribute('interface', feed_uri)
 301 if icon_uri:
 302         icon = add_node(root, 'icon')
 303         icon.setAttribute('href', icon_uri)
 304         icon.setAttribute('type', 'image/png')
 305 if homepage:
 306         add_node(root, 'homepage', homepage)
 307 if category:
 308         add_node(root, 'category', category)
 309
 310 package = add_node(root, 'package-implementation', '')
 311 package.setAttribute('package', pkg_name)
 312
 313 group = add_node(root, 'group', '')
 314 group.setAttribute('arch', pkg_arch)
 315 group.setAttribute('license', license)
 316
 317 for req in requires:
 318         req_element = add_node(group, 'requires', before = '\n    ', after = '')
 319         req_element.setAttribute('interface', req.interface)
 320         binding = add_node(req_element, 'environment', before = '\n      ', after = '\n    ')
 321         binding.setAttribute('name', 'LD_LIBRARY_PATH')
 322         binding.setAttribute('insert', 'usr/lib')
 323
 324 if pkg_main:
 325         group.setAttribute('main', pkg_main)
 326         package.setAttribute('main', '/' + pkg_main)
 327
 328 impl = add_node(group, 'implementation', before = '\n    ', after = '\n  ')
 329 impl.setAttribute('id', digest)
 330 impl.setAttribute('version', pkg_version)
 331 impl.setAttribute('released', time.strftime('%Y-%m-%d', time.localtime(buildtime)))
 332
 333 archive = add_node(impl, 'archive', before = '\n      ', after = '\n    ')
 334 archive.setAttribute('href', pkg_url)
 335 archive.setAttribute('size', str(os.path.getsize(rpm_file)))
 336
 337 # Add our new version to the main feed...
 338
 339 output_stream = tempfile.NamedTemporaryFile(prefix = 'rpm2zero-')
 340 try:
 341         output_stream.write("<?xml version='1.0'?>\n")
 342         root.writexml(output_stream)
 343         output_stream.write('\n')
 344         output_stream.flush()
 345
 346         publishing_options = []
 347         if options.key:
 348                 # Note: 0publish < 0.16 requires the --xmlsign option too
 349                 publishing_options += ['--xmlsign', '--key', options.key]
 350         check_call([os.environ['PUBLISH_COMMAND']] + publishing_options + ['--local', output_stream.name, target_feed_file])
 351         print "Added version %s to %s" % (pkg_version, target_feed_file)
 352 finally:
 353         output_stream.close()