scripts/mk-tarball-moduleset.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import stat
   6 import re
   7 import md5
   8 import getopt
   9 import urlparse
  10 import ConfigParser
  11 import xml.dom.minidom
  12
  13 usage = 'mk-tarball-moduleset -d DEPS -u URI -s DIR'
  14 help = \
  15 '''Build a module set from a list of dependencies, a directory of tarballs
  16 and a URI base.
  17
  18 Options:
  19   -d, --dependencies=FILE   The list of dependencies
  20   -u, --uri=URI             The base URI for the tarballs
  21   -s, --source=DIR          Location of tarballs
  22   -x, --exceptions=FILE     A file including exceptions for sources'''
  23
  24 def read_deps(filename):
  25     deps = []
  26     deps_dict = {}
  27     fp = open(filename)
  28     line = fp.readline()
  29     while line:
  30         pkg, dep_pkgs = line.split(':', 1)
  31         assert not deps_dict.has_key(pkg), '%s repeated' % pkg
  32         dep_pkgs = [ dep.strip() for dep in dep_pkgs.split() ]
  33         deps.append((pkg, dep_pkgs))
  34         deps_dict[pkg] = dep_pkgs
  35         line = fp.readline()
  36     # verify that all dependencies are listed
  37     for pkg in deps_dict.keys():
  38         for dep in deps_dict[pkg]:
  39             assert deps_dict.has_key(dep), 'dependency %s not found' % dep
  40     return deps
  41
  42 class SourceRepo:
  43     def __init__(self, sourcedir, uribase, exceptions):
  44         self.sourcedir = sourcedir
  45         self.uribase = uribase
  46         self.exceptions = exceptions
  47
  48     def _find_tarball(self, pkg):
  49         '''Find the first file in sourcedir that looks like a tarball for
  50         the given package.  Bzip2 compressed tarballs are preferred.'''
  51         files = os.listdir(self.sourcedir)
  52         pat = re.compile(r'%s-([0-9].*)\.tar\.bz2' % pkg.replace('+', '\\+'))
  53         for filename in files:
  54             match = pat.match(filename)
  55             if match:
  56                 return filename, match.group(1)
  57         pat = re.compile(r'%s-([0-9].*)\.tar\.gz' % pkg.replace('+', '\\+'))
  58         for filename in files:
  59             match = pat.match(filename)
  60             if match:
  61                 return filename, match.group(1)
  62         raise RuntimeError('no file found for package %s' % pkg)
  63
  64     def create_tarball_node(self, document, pkg):
  65         tarball = document.createElement('tarball')
  66         tarball.setAttribute('id', pkg)
  67         tarball.appendChild(document.createTextNode('\n'))
  68         source_node = document.createElement('source')
  69         tarball.appendChild(source_node)
  70         tarball.appendChild(document.createTextNode('\n'))
  71
  72         if self.exceptions.has_section(pkg):
  73             tarball.setAttribute('version',
  74                                  self.exceptions.get(pkg, 'version'))
  75
  76             source_node.setAttribute('href',
  77                                      self.exceptions.get(pkg, 'href'))
  78             source_node.setAttribute('size',
  79                                      self.exceptions.get(pkg, 'size'))
  80             source_node.setAttribute('md5sum',
  81                                      self.exceptions.get(pkg, 'md5sum'))
  82         else:
  83             filename, version = self._find_tarball(pkg)
  84             tarball.setAttribute('version', version)
  85
  86             source_node.setAttribute('href',
  87                                      urlparse.urljoin(self.uribase, filename))
  88             info = os.stat(os.path.join(self.sourcedir, filename))
  89             size = info[stat.ST_SIZE]
  90             source_node.setAttribute('size', str(info[stat.ST_SIZE]))
  91
  92             sum = md5.new()
  93             fp = open(os.path.join(self.sourcedir, filename), 'rb')
  94             data = fp.read(4096)
  95             while data:
  96                 sum.update(data)
  97                 data = fp.read(4096)
  98             fp.close()
  99             source_node.setAttribute('md5sum', sum.hexdigest())
 100         return tarball
 101
 102 def main(args):
 103     try:
 104         opts, args = getopt.getopt(args, 'd:u:s:x:h',
 105                                    ['dependencies=', 'uri=', 'source=',
 106                                     'exceptions=', 'help'])
 107     except getopt.error, exc:
 108         sys.stderr.write('mk-tarball-moduleset: %s\n' % str(exc))
 109         sys.stderr.write(usage + '\n')
 110         sys.exit(1)
 111
 112     dependencies = None
 113     uri = None
 114     source = None
 115     exceptions = ConfigParser.ConfigParser()
 116     for opt, arg in opts:
 117         if opt in ('-h', '--help'):
 118             print usage
 119             print help
 120             sys.exit(0)
 121         elif opt in ('-d', '--dependencies'):
 122             dependencies = arg
 123         elif opt in ('-u', '--uri'):
 124             uri = arg
 125         elif opt in ('-s', '--source'):
 126             source = arg
 127         elif opt in ('-x', '--exceptions'):
 128             exceptions.read(arg)
 129     if not dependencies or not uri or not source:
 130         sys.stderr.write(usage + '\n')
 131         sys.exit(1)
 132
 133     repo = SourceRepo(source, uri, exceptions)
 134     deps = read_deps(dependencies)
 135
 136     document = xml.dom.minidom.Document()
 137     document.appendChild(document.createElement('moduleset'))
 138     document.documentElement.appendChild(document.createTextNode('\n'))
 139     for (pkg, dep_pkgs) in deps:
 140         if pkg.startswith('meta-'):
 141             pkg_node = document.createElement('metamodule')
 142             pkg_node.setAttribute('id', pkg)
 143             pkg_node.appendChild(document.createTextNode('\n'))
 144         else:
 145             pkg_node = repo.create_tarball_node(document, pkg)
 146         if dep_pkgs:
 147             deps = document.createElement('dependencies')
 148             deps.appendChild(document.createTextNode('\n'))
 149             for dep_pkg in dep_pkgs:
 150                 node = document.createElement('dep')
 151                 node.setAttribute('package', dep_pkg)
 152                 deps.appendChild(node)
 153                 deps.appendChild(document.createTextNode('\n'))
 154             pkg_node.appendChild(deps)
 155             pkg_node.appendChild(document.createTextNode('\n'))
 156
 157         document.documentElement.appendChild(pkg_node)
 158         document.documentElement.appendChild(document.createTextNode('\n'))
 159
 160     document.writexml(sys.stdout)
 161     document.unlink()
 162
 163 if __name__ == '__main__':
 164     main(sys.argv[1:])