pkg2zero

   1 #!/usr/bin/env python
   2 # Copyright (C) 2009, Thomas Leonard
   3 # Copyright (C) 2008, Anders F Bjorklund
   4 # See the COPYING file for details, or visit http://0install.net.
   5
   6 import sys, time
   7 from optparse import OptionParser
   8 import tempfile, shutil, os
   9 from xml.dom import minidom
  10 import gzip
  11 try:
  12         import xml.etree.cElementTree as ET # Python 2.5
  13 except ImportError:
  14         try:
  15                 import xml.etree.ElementTree as ET
  16         except ImportError:
  17                 try:
  18                         import cElementTree as ET # http://effbot.org
  19                 except ImportError:
  20                         import elementtree.ElementTree as ET
  21
  22 import subprocess
  23 try:
  24         from subprocess import check_call
  25 except ImportError:
  26         def check_call(*popenargs, **kwargs):
  27                 rc = subprocess.call(*popenargs, **kwargs)
  28                 if rc != 0: raise OSError, rc
  29
  30 from zeroinstall.injector import model, qdom, distro
  31 from zeroinstall.zerostore import unpack
  32
  33 from support import read_child, add_node, Mappings
  34
  35 manifest_algorithm = 'sha1new'
  36
  37 deb_category_to_freedesktop = {
  38         'devel' : 'Development',
  39         'web' : 'Network',
  40         'graphics' : 'Graphics',
  41         'games' : 'Game',
  42 }
  43
  44 rpm_group_to_freedesktop = {
  45         'Development/Libraries' : 'Development',
  46 }
  47
  48 valid_categories = [
  49         'AudioVideo',
  50         'Audio',
  51         'Video',
  52         'Development',
  53         'Education',
  54         'Game',
  55         'Graphics',
  56         'Network',
  57         'Office',
  58         'Settings',
  59         'System',
  60         'Utility',
  61 ]
  62
  63 # Parse command-line arguments
  64
  65 parser = OptionParser('usage: %prog [options] http://.../package.deb [target-feed.xml]\n'
  66                       '       %prog [options] http://.../package.rpm [target-feed.xml]\n'
  67                       '       %prog [options] package-name [target-feed.xml]\n'
  68                       'Publish a Debian or RPM package in a Zero Install feed.\n'
  69                       "target-feed.xml is created if it doesn't already exist.")
  70 parser.add_option("-a", "--archive-url", help="archive to use as the package contents")
  71 parser.add_option("", "--archive-extract", help="only extract files under this subdirectory")
  72 parser.add_option("", "--license", help="value for 'license' attribute")
  73 parser.add_option("-r", "--repomd-file", help="repository metadata file")
  74 parser.add_option("", "--path", help="location of packages [5/os/i386]")
  75 parser.add_option("-p", "--packages-file", help="Debian package index file")
  76 parser.add_option("-m", "--mirror", help="location of packages [http://ftp.debian.org/debian] or [http://mirror.centos.org/centos]")
  77 parser.add_option("-k", "--key", help="key to use for signing")
  78 (options, args) = parser.parse_args()
  79
  80 if len(args) < 1 or len(args) > 2:
  81         parser.print_help()
  82         sys.exit(1)
  83
  84 # Load dependency mappings
  85 mappings = Mappings()
  86
  87 class Package:
  88         name = '(unknown)'
  89         version = None
  90         arch = None
  91         category = None
  92         homepage = None
  93         buildtime = None
  94         license = None
  95
  96         def __init__(self):
  97                 self.requires = []
  98
  99 class DebRepo:
 100         def __init__(self, options):
 101                 self.packages_base_url = (options.mirror or 'http://ftp.debian.org/debian') + '/'
 102                 self.packages_file = options.packages_file or 'Packages'
 103
 104         def get_repo_metadata(self, pkg_name):
 105                 if not os.path.isfile(self.packages_file):
 106                         print >>sys.stderr, ("File '%s' not found (use -p to give its location).\n"
 107                                 "Either download one (e.g. ftp://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.bz2),\n"
 108                                 "or specify the full URL of the .deb package to use.") % self.packages_file
 109                         sys.exit(1)
 110                 if self.packages_file.endswith('.bz2'):
 111                         import bz2
 112                         opener = bz2.BZ2File
 113                 else:
 114                         opener = file
 115                 pkg_data = "\n" + opener(self.packages_file).read()
 116                 try:
 117                         i = pkg_data.index('\nPackage: %s\n' % pkg_name)
 118                 except ValueError:
 119                         raise Exception("Package '%s' not found in Packages file '%s'." % (pkg_name, self.packages_file))
 120                 j = pkg_data.find('\n\n', i)
 121                 if j == -1:
 122                         pkg_info = pkg_data[i:]
 123                 else:
 124                         pkg_info = pkg_data[i:j]
 125                 filename = None
 126                 digest = {}
 127                 for line in pkg_info.split('\n'):
 128                         if ':' in line and not line.startswith(' '):
 129                                 key, value = line.split(':', 1)
 130                                 if key == 'Filename':
 131                                         filename = value.strip()
 132                                 elif key in ('SHA1', 'SHA256'):
 133                                         digest[key.lower()] = value.strip()
 134                 if filename is None:
 135                         raise Exception('Filename: field not found in package data:\n' + pkg_info)
 136                 pkg_url = self.packages_base_url + filename
 137
 138                 return pkg_url, digest
 139
 140         def get_package_metadata(self, pkg_file):
 141                 package = Package()
 142
 143                 details = read_child(['dpkg-deb', '--info', pkg_file])
 144
 145                 description_and_summary = details.split('\n Description: ')[1].split('\n')
 146                 package.summary = description_and_summary[0]
 147                 description = ''
 148                 for x in description_and_summary[1:]:
 149                         if not x: continue
 150                         assert x[0] == ' '
 151                         x = x[1:]
 152                         if x[0] != ' ':
 153                                 break
 154                         if x == ' .':
 155                                 description += '\n'
 156                         else:
 157                                 description += x[1:].replace('.  ', '. ') + '\n'
 158                 package.description = description.strip()
 159
 160                 for line in details.split('\n'):
 161                         if not line: continue
 162                         assert line.startswith(' ')
 163                         line = line[1:]
 164                         if ':' in line:
 165                                 key, value = line.split(':', 1)
 166                                 value = value.strip()
 167                                 if key == 'Section':
 168                                         package.category = deb_category_to_freedesktop.get(value)
 169                                         if not package.category:
 170                                                 if value != 'libs':
 171                                                         print >>sys.stderr, "Warning: no mapping for Debian category '%s'" % value
 172                                 elif key == 'Package':
 173                                         package.name = value
 174                                 elif key == 'Version':
 175                                         value = value.replace('cvs', '')
 176                                         value = value.replace('svn', '')
 177                                         if ':' in value: value = value.split(':', 1)[1]
 178                                         package.version = distro.try_cleanup_distro_version(value)
 179                                 elif key == 'Architecture':
 180                                         if '-' in value:
 181                                                 arch, value = value.split('-', 1)
 182                                         else:
 183                                                 arch = 'linux'
 184                                         if value == 'amd64':
 185                                                 value = 'x86_64'
 186                                         elif value == 'all':
 187                                                 value = '*'
 188                                         package.arch = arch.capitalize() + '-' + value
 189                                 elif key == 'Depends':
 190                                         for x in value.split(','):
 191                                                 req = mappings.process(x)
 192                                                 if req:
 193                                                         package.requires.append(req)
 194                 return package
 195
 196 class RPMRepo:
 197         def __init__(self, options):
 198                 self.packages_base_url = (options.mirror or 'http://mirror.centos.org/centos') + '/'
 199                 self.packages_base_dir = (options.path or '5/os/i386') + '/'
 200                 self.repomd_file = options.repomd_file or 'repodata/repomd.xml'
 201                 if not os.path.isfile(self.repomd_file):
 202                         print >>sys.stderr, ("File '%s' not found (use -r to give its location).\n"
 203                                 "Either download one (e.g. http://mirror.centos.org/centos/5/os/i386/repodata/repomd.xml),\n"
 204                                 "or specify the full URL of the .rpm package to use.") % self.repomd_file
 205                         sys.exit(1)
 206
 207         def get_repo_metadata(self, pkg_name):
 208                 primary_file = None
 209                 repomd = minidom.parse(self.repomd_file)
 210                 repo_top = os.path.dirname(os.path.dirname(self.repomd_file))
 211                 for data in repomd.getElementsByTagName("data"):
 212                         if data.attributes["type"].nodeValue == "primary":
 213                                 for node in data.getElementsByTagName("location"):
 214                                         primary_file = os.path.join(repo_top, node.attributes["href"].nodeValue)
 215                 location = None
 216                 primary = ET.parse(gzip.open(primary_file))
 217                 NS = "http://linux.duke.edu/metadata/common"
 218                 metadata = primary.getroot()
 219                 pkg_data = None
 220                 for package in metadata.findall("{%s}package" % NS):
 221                         if package.find("{%s}name" % NS).text == pkg_name:
 222                                 pkg_data = package
 223                                 location = pkg_data.find("{%s}location" % NS).get("href")
 224                                 break
 225                 if pkg_data is None:
 226                         raise Exception("Package '%s' not found in repodata." % pkg_name)
 227                 checksum = pkg_data.find("{%s}checksum" % NS)
 228                 digest = {}
 229                 if checksum.get("type") == "sha":
 230                         digest["sha1"] = checksum.text
 231                 if checksum.get("type") == "sha256":
 232                         digest["sha256"] = checksum.text
 233                 if location is None:
 234                         raise Exception('location tag not found in primary metadata:\n' + primary_file)
 235                 pkg_url = self.packages_base_url + self.packages_base_dir + location
 236
 237                 return pkg_url, digest
 238
 239         def get_package_metadata(self, pkg_file):
 240                 package = Package()
 241
 242                 query_format = '%{SUMMARY}\\a%{DESCRIPTION}\\a%{NAME}\\a%{VERSION}\\a%{OS}\\a%{ARCH}\\a%{URL}\\a%{GROUP}\\a%{LICENSE}\\a%{BUILDTIME}\\a[%{REQUIRES}\\n]'
 243                 headers = read_child(['rpm', '--qf', query_format, '-qp', pkg_file]).split('\a')
 244
 245                 package.summary = headers[0].strip()
 246                 package.description = headers[1].strip()
 247
 248                 package.name = headers[2]
 249                 value = headers[3]
 250                 value = value.replace('cvs', '')
 251                 value = value.replace('svn', '')
 252                 value = distro.try_cleanup_distro_version(value)
 253                 package.version = value
 254                 value = headers[4]
 255                 package.arch = value.capitalize()
 256                 value = headers[5]
 257                 if value == 'amd64':
 258                         value = 'x86_64'
 259                 if value == 'noarch':
 260                         value = '*'
 261                 package.arch += '-' + value
 262                 value = headers[6].strip()
 263                 package.page = value
 264                 category = None
 265                 value = headers[7].strip()
 266                 package.category = rpm_group_to_freedesktop.get(value)
 267                 if not category:
 268                         print >>sys.stderr, "Warning: no mapping for RPM group '%s'" % value
 269
 270                 value = headers[8].strip()
 271                 package.license = value
 272                 value = headers[9].strip()
 273                 package.buildtime = long(value)
 274                 value = headers[10].strip()
 275                 for x in value.split('\n'):
 276                         if x.startswith('rpmlib'):
 277                                 continue
 278                         req = mappings.process(x)
 279                         if req:
 280                                 package.requires.append(req)
 281                 return package
 282
 283 if args[0].endswith('.deb') or options.packages_file:
 284         repo = DebRepo(options)
 285 elif args[0].endswith('.rpm') or options.repomd_file:
 286         repo = RPMRepo(options)
 287 else:
 288         print >>sys.stderr, "Use --packages-file for Debian, or --repomd-file for RPM"
 289         sys.exit(1)
 290
 291 pkg_data = None
 292
 293 if options.archive_url:
 294         pkg_file = os.path.abspath(args[0])
 295         archive_url = options.archive_url
 296         archive_file = os.path.abspath(archive_url.rsplit('/', 1)[1])
 297         digest = {}
 298         assert os.path.exists(pkg_file), ("%s doesn't exist!" % pkg_file)
 299 else:
 300         scheme = args[0].split(':', 1)[0]
 301         if scheme in ('http', 'https', 'ftp'):
 302                 archive_url = args[0]
 303                 digest = {}
 304         else:
 305                 archive_url, digest = repo.get_repo_metadata(args[0])
 306         archive_file = pkg_file = os.path.abspath(archive_url.rsplit('/', 1)[1])
 307
 308 # pkg_url, pkg_archive = .deb or .rpm with the metadata
 309 # archive_url, archive_file = .dep, .rpm or .tar.bz2 with the contents
 310 #
 311 # Often pkg == archive, but sometimes it's useful to convert packages to tarballs
 312 # so people don't need special tools to extract them.
 313
 314
 315 # Download package, if required
 316
 317 if not os.path.exists(pkg_file):
 318         print >>sys.stderr, "File '%s' not found, so downloading from %s..." % (pkg_file, archive_url)
 319         file_part = pkg_file + '.part'
 320         check_call(['wget', '-O', file_part, archive_url])
 321         os.rename(file_part, pkg_file)
 322
 323 # Check digest, if known
 324
 325 if "sha256" in digest:
 326         import hashlib
 327         m = hashlib.new('sha256')
 328         expected_digest = digest["sha256"]
 329 elif "sha1" in digest:
 330         try:
 331                 import hashlib
 332                 m = hashlib.new('sha1')
 333         except ImportError:
 334                 import sha
 335                 m = sha.new()
 336         expected_digest = digest["sha1"]
 337 else:
 338         m = None
 339
 340 if m:
 341         m.update(file(archive_file).read())
 342         actual = m.hexdigest()
 343         if actual != expected_digest:
 344                 raise Exception("Incorrect digest on package file! Was " + actual + ", but expected " + expected_digest)
 345         else:
 346                 print "Package's digest matches value in reposistory metadata (" + actual + "). Good."
 347 else:
 348         print >>sys.stderr, "Note: no SHA-1 or SHA-256 digest known for this package, so not checking..."
 349
 350 # Extract meta-data from package
 351
 352 pkg_metadata = repo.get_package_metadata(pkg_file)
 353
 354 # Unpack package, find binaries and .desktop files, and add to cache
 355
 356 possible_mains = []
 357 icondata = None
 358 tmp = tempfile.mkdtemp(prefix = 'pkg2zero-')
 359 try:
 360         unpack_dir = tmp
 361         unpack.unpack_archive(archive_file, open(archive_file), destdir = unpack_dir, extract = options.archive_extract)
 362         if options.archive_extract:
 363                 unpack_dir = os.path.join(unpack_dir, options.archive_extract)
 364
 365         icon = None
 366         images = {}
 367         for root, dirs, files in os.walk(unpack_dir):
 368                 assert root.startswith(unpack_dir)
 369                 relative_root = root[len(unpack_dir) + 1:]
 370                 for name in files:
 371                         full = os.path.join(root, name)
 372                         f = os.path.join(relative_root, name)
 373                         print f
 374                         if f.endswith('.desktop'):
 375                                 for line in file(full):
 376                                         if line.startswith('Categories'):
 377                                                 for cat in line.split('=', 1)[1].split(';'):
 378                                                         cat = cat.strip()
 379                                                         if cat in valid_categories:
 380                                                                 category = cat
 381                                                                 break
 382                                         elif line.startswith('Icon'):
 383                                                 icon = line.split('=', 1)[1].strip()
 384                         elif f.startswith('bin/') or f.startswith('usr/bin/') or f.startswith('usr/games/'):
 385                                 if os.path.isfile(full):
 386                                         possible_mains.append(f)
 387                         elif f.endswith('.png'):
 388                                 images[f] = full
 389                                 images[os.path.basename(f)] = full
 390                                 # make sure to also map basename without the extension
 391                                 images[os.path.splitext(os.path.basename(f))[0]] = full
 392
 393         icondata = None
 394         if icon in images:
 395                 print "Using %s for icon" % os.path.basename(images[icon])
 396                 icondata = file(images[icon]).read()
 397
 398         manifest = read_child(['0store', 'manifest', unpack_dir, manifest_algorithm])
 399         digest = manifest.rsplit('\n', 2)[1]
 400         check_call(['0store', 'add', digest, unpack_dir])
 401 finally:
 402         shutil.rmtree(tmp)
 403
 404 if possible_mains:
 405         possible_mains = sorted(possible_mains, key = len)
 406         pkg_main = possible_mains[0]
 407         if len(possible_mains) > 1:
 408                 print "Warning: several possible main binaries found:"
 409                 print "- " + pkg_main + " (I chose this one)"
 410                 for x in possible_mains[1:]:
 411                         print "- " + x
 412 else:
 413         pkg_main = None
 414
 415 # Make sure we haven't added this version already...
 416
 417 if len(args) > 1:
 418         target_feed_file = args[1]
 419         target_icon_file = args[1].replace('.xml', '.png')
 420 else:
 421         target_feed_file = pkg_metadata.name + '.xml'
 422         target_icon_file = pkg_metadata.name + '.png'
 423
 424 feed_uri = None
 425 icon_uri = None
 426 if os.path.isfile(target_feed_file):
 427         dom = qdom.parse(file(target_feed_file))
 428         old_target_feed = model.ZeroInstallFeed(dom, local_path = target_feed_file)
 429         existing_impl = old_target_feed.implementations.get(digest)
 430         if existing_impl:
 431                 print >>sys.stderr, ("Feed '%s' already contains an implementation with this digest!\n%s" % (target_feed_file, existing_impl))
 432                 sys.exit(1)
 433 else:
 434         # No target, so need to pick a URI
 435         feed_uri = mappings.lookup(pkg_metadata.name)
 436         if feed_uri is None:
 437                 suggestion = mappings.get_suggestion(pkg_metadata.name)
 438                 uri = raw_input('Enter the URI for this feed [%s]: ' % suggestion).strip()
 439                 if not uri:
 440                         uri = suggestion
 441                 assert uri.startswith('http://') or uri.startswith('https://') or uri.startswith('ftp://'), uri
 442                 feed_uri = uri
 443                 mappings.add_mapping(pkg_metadata.name, uri)
 444
 445         if icondata and not os.path.isfile(target_icon_file):
 446                 file = open(target_icon_file, 'wb')
 447                 file.write(icondata)
 448                 file.close()
 449         if icon_uri is None:
 450                 suggestion = 'http://0install.net/feed_icons/' + target_icon_file
 451                 uri = raw_input('Enter the URI for this icon [%s]: ' % suggestion).strip()
 452                 if not uri:
 453                         uri = suggestion
 454                 assert uri.startswith('http://') or uri.startswith('https://') or uri.startswith('ftp://'), uri
 455                 icon_uri = uri
 456
 457 # Create a local feed with just the new version...
 458
 459 template = '''<interface xmlns="http://zero-install.sourceforge.net/2004/injector/interface">
 460 </interface>'''
 461 doc = minidom.parseString(template)
 462 root = doc.documentElement
 463
 464 add_node(root, 'name', pkg_metadata.name)
 465 add_node(root, 'summary', pkg_metadata.summary)
 466 add_node(root, 'description', pkg_metadata.description)
 467 feed_for = add_node(root, 'feed-for', '')
 468 if feed_uri:
 469         feed_for.setAttribute('interface', feed_uri)
 470 if icon_uri:
 471         icon = add_node(root, 'icon')
 472         icon.setAttribute('href', icon_uri)
 473         icon.setAttribute('type', 'image/png')
 474 if pkg_metadata.homepage:
 475         add_node(root, 'homepage', pkg_metadata.homepage)
 476 if pkg_metadata.category:
 477         add_node(root, 'category', pkg_metadata.category)
 478
 479 package = add_node(root, 'package-implementation', '')
 480 package.setAttribute('package', pkg_metadata.name)
 481
 482 group = add_node(root, 'group', '')
 483 if pkg_metadata.arch:
 484         group.setAttribute('arch', pkg_metadata.arch)
 485 else:
 486         print >>sys.stderr, "No Architecture: field in package"
 487 if pkg_metadata.license:
 488         group.setAttribute('license', pkg_metadata.license)
 489
 490 for req in pkg_metadata.requires:
 491         req_element = add_node(group, 'requires', before = '\n    ', after = '')
 492         req_element.setAttribute('interface', req.interface)
 493         binding = add_node(req_element, 'environment', before = '\n      ', after = '\n    ')
 494         binding.setAttribute('name', 'LD_LIBRARY_PATH')
 495         binding.setAttribute('insert', 'usr/lib')
 496
 497 if pkg_main:
 498         group.setAttribute('main', pkg_main)
 499         package.setAttribute('main', '/' + pkg_main)
 500
 501 impl = add_node(group, 'implementation', before = '\n    ', after = '\n  ')
 502 impl.setAttribute('id', digest)
 503 assert pkg_metadata.version
 504 impl.setAttribute('version', pkg_metadata.version)
 505
 506 if options.license:
 507         impl.setAttribute('license', options.license)
 508
 509 if pkg_metadata.buildtime:
 510         impl.setAttribute('released', time.strftime('%Y-%m-%d', time.localtime(pkg_metadata.buildtime)))
 511 else:
 512         impl.setAttribute('released', time.strftime('%Y-%m-%d'))
 513
 514 archive = add_node(impl, 'archive', before = '\n      ', after = '\n    ')
 515 archive.setAttribute('href', archive_url)
 516 archive.setAttribute('size', str(os.path.getsize(archive_file)))
 517 if options.archive_extract:
 518         archive.setAttribute('extract', options.archive_extract)
 519
 520 # Add our new version to the main feed...
 521
 522 output_stream = tempfile.NamedTemporaryFile(prefix = 'pkg2zero-')
 523 try:
 524         output_stream.write("<?xml version='1.0'?>\n")
 525         root.writexml(output_stream)
 526         output_stream.write('\n')
 527         output_stream.flush()
 528
 529         publishing_options = []
 530         if options.key:
 531                 # Note: 0publish < 0.16 requires the --xmlsign option too
 532                 publishing_options += ['--xmlsign', '--key', options.key]
 533         check_call([os.environ['PUBLISH_COMMAND']] + publishing_options + ['--local', output_stream.name, target_feed_file])
 534         print "Added version %s to %s" % (pkg_metadata.version, target_feed_file)
 535 finally:
 536         output_stream.close()