testing/mozbase/setup_development.py

   1 #!/usr/bin/env python
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5 # You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 """
   8 Setup mozbase packages for development.
   9
  10 Packages may be specified as command line arguments.
  11 If no arguments are given, install all packages.
  12
  13 See https://wiki.mozilla.org/Auto-tools/Projects/Mozbase
  14 """
  15
  16 import os
  17 import subprocess
  18 import sys
  19 from optparse import OptionParser
  20 from subprocess import PIPE
  21 try:
  22     from subprocess import check_call as call
  23 except ImportError:
  24     from subprocess import call
  25
  26
  27 # directory containing this file
  28 here = os.path.dirname(os.path.abspath(__file__))
  29
  30 # all python packages
  31 mozbase_packages = [i for i in os.listdir(here)
  32                     if os.path.exists(os.path.join(here, i, 'setup.py'))]
  33 test_packages = [ "mock" # testing: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Tests
  34                   ]
  35 extra_packages = [ "sphinx" # documentation: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Documentation
  36                   ]
  37
  38 def cycle_check(order, dependencies):
  39     """ensure no cyclic dependencies"""
  40     order_dict = dict([(j, i) for i, j in enumerate(order)])
  41     for package, deps in dependencies.items():
  42         index = order_dict[package]
  43         for d in deps:
  44             assert index > order_dict[d], "Cyclic dependencies detected"
  45
  46 def info(directory):
  47     "get the package setup.py information"
  48
  49     assert os.path.exists(os.path.join(directory, 'setup.py'))
  50
  51     # setup the egg info
  52     try:
  53         call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE)
  54     except subprocess.CalledProcessError:
  55         print "Error running setup.py in %s" % directory
  56         raise
  57
  58     # get the .egg-info directory
  59     egg_info = [entry for entry in os.listdir(directory)
  60                 if entry.endswith('.egg-info')]
  61     assert len(egg_info) == 1, 'Expected one .egg-info directory in %s, got: %s' % (directory, egg_info)
  62     egg_info = os.path.join(directory, egg_info[0])
  63     assert os.path.isdir(egg_info), "%s is not a directory" % egg_info
  64
  65     # read the package information
  66     pkg_info = os.path.join(egg_info, 'PKG-INFO')
  67     info_dict = {}
  68     for line in file(pkg_info).readlines():
  69         if not line or line[0].isspace():
  70             continue # XXX neglects description
  71         assert ':' in line
  72         key, value = [i.strip() for i in line.split(':', 1)]
  73         info_dict[key] = value
  74
  75     return info_dict
  76
  77 def get_dependencies(directory):
  78     "returns the package name and dependencies given a package directory"
  79
  80     # get the package metadata
  81     info_dict = info(directory)
  82
  83     # get the .egg-info directory
  84     egg_info = [entry for entry in os.listdir(directory)
  85                 if entry.endswith('.egg-info')][0]
  86
  87     # read the dependencies
  88     requires = os.path.join(directory, egg_info, 'requires.txt')
  89     if os.path.exists(requires):
  90         dependencies = [line.strip()
  91                         for line in file(requires).readlines()
  92                         if line.strip()]
  93     else:
  94         dependencies = []
  95
  96     # return the information
  97     return info_dict['Name'], dependencies
  98
  99 def dependency_info(dep):
 100     "return dictionary of dependency information from a dependency string"
 101     retval = dict(Name=None, Type=None, Version=None)
 102     for joiner in ('==', '<=', '>='):
 103         if joiner in dep:
 104             retval['Type'] = joiner
 105             name, version = [i.strip() for i in dep.split(joiner, 1)]
 106             retval['Name'] = name
 107             retval['Version'] = version
 108             break
 109     else:
 110         retval['Name'] = dep.strip()
 111     return retval
 112
 113 def unroll_dependencies(dependencies):
 114     """
 115     unroll a set of dependencies to a flat list
 116
 117     dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']),
 118                     'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']),
 119                     'packageC': set(['packageE']),
 120                     'packageE': set(['packageF', 'packageG']),
 121                     'packageF': set(['packageG']),
 122                     'packageX': set(['packageA', 'packageG'])}
 123     """
 124
 125     order = []
 126
 127     # flatten all
 128     packages = set(dependencies.keys())
 129     for deps in dependencies.values():
 130         packages.update(deps)
 131
 132     while len(order) != len(packages):
 133
 134         for package in packages.difference(order):
 135             if set(dependencies.get(package, set())).issubset(order):
 136                 order.append(package)
 137                 break
 138         else:
 139             raise AssertionError("Cyclic dependencies detected")
 140
 141     cycle_check(order, dependencies) # sanity check
 142
 143     return order
 144
 145
 146 def main(args=sys.argv[1:]):
 147
 148     # parse command line options
 149     usage = '%prog [options] [package] [package] [...]'
 150     parser = OptionParser(usage=usage, description=__doc__)
 151     parser.add_option('-d', '--dependencies', dest='list_dependencies',
 152                       action='store_true', default=False,
 153                       help="list dependencies for the packages")
 154     parser.add_option('--list', action='store_true', default=False,
 155                       help="list what will be installed")
 156     parser.add_option('--extra', '--install-extra-packages', action='store_true', default=False,
 157                       help="installs extra supporting packages as well as core mozbase ones")
 158     options, packages = parser.parse_args(args)
 159
 160     if not packages:
 161         # install all packages
 162         packages = sorted(mozbase_packages)
 163
 164     # ensure specified packages are in the list
 165     assert set(packages).issubset(mozbase_packages), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages)
 166
 167     if options.list_dependencies:
 168         # list the package dependencies
 169         for package in packages:
 170             print '%s: %s' % get_dependencies(os.path.join(here, package))
 171         parser.exit()
 172
 173     # gather dependencies
 174     # TODO: version conflict checking
 175     deps = {}
 176     alldeps = {}
 177     mapping = {} # mapping from subdir name to package name
 178     # core dependencies
 179     for package in packages:
 180         key, value = get_dependencies(os.path.join(here, package))
 181         deps[key] = [dependency_info(dep)['Name'] for dep in value]
 182         mapping[package] = key
 183
 184         # keep track of all dependencies for non-mozbase packages
 185         for dep in value:
 186             alldeps[dependency_info(dep)['Name']] = ''.join(dep.split())
 187
 188     # indirect dependencies
 189     flag = True
 190     while flag:
 191         flag = False
 192         for value in deps.values():
 193             for dep in value:
 194                 if dep in mozbase_packages and dep not in deps:
 195                     key, value = get_dependencies(os.path.join(here, dep))
 196                     deps[key] = [dep for dep in value]
 197
 198                     for dep in value:
 199                         alldeps[dep] = ''.join(dep.split())
 200                     mapping[package] = key
 201                     flag = True
 202                     break
 203             if flag:
 204                 break
 205
 206     # get the remaining names for the mapping
 207     for package in mozbase_packages:
 208         if package in mapping:
 209             continue
 210         key, value = get_dependencies(os.path.join(here, package))
 211         mapping[package] = key
 212
 213     # unroll dependencies
 214     unrolled = unroll_dependencies(deps)
 215
 216     # make a reverse mapping: package name -> subdirectory
 217     reverse_mapping = dict([(j,i) for i, j in mapping.items()])
 218
 219     # we only care about dependencies in mozbase
 220     unrolled = [package for package in unrolled if package in reverse_mapping]
 221
 222     if options.list:
 223         # list what will be installed
 224         for package in unrolled:
 225             print package
 226         parser.exit()
 227
 228     # set up the packages for development
 229     for package in unrolled:
 230         call([sys.executable, 'setup.py', 'develop', '--no-deps'],
 231              cwd=os.path.join(here, reverse_mapping[package]))
 232
 233     # add the directory of sys.executable to path to aid the correct
 234     # `easy_install` getting called
 235     # https://bugzilla.mozilla.org/show_bug.cgi?id=893878
 236     os.environ['PATH'] = '%s%s%s' % (os.path.dirname(os.path.abspath(sys.executable)),
 237                                      os.path.pathsep,
 238                                      os.environ.get('PATH', '').strip(os.path.pathsep))
 239
 240     # install non-mozbase dependencies
 241     # these need to be installed separately and the --no-deps flag
 242     # subsequently used due to a bug in setuptools; see
 243     # https://bugzilla.mozilla.org/show_bug.cgi?id=759836
 244     pypi_deps = dict([(i, j) for i,j in alldeps.items()
 245                       if i not in unrolled])
 246     for package, version in pypi_deps.items():
 247         # easy_install should be available since we rely on setuptools
 248         call(['easy_install', version])
 249
 250     # install packages required for unit testing
 251     for package in test_packages:
 252         call(['easy_install', package])
 253
 254     # install extra non-mozbase packages if desired
 255     if options.extra:
 256         for package in extra_packages:
 257             call(['easy_install', package])
 258
 259 if __name__ == '__main__':
 260     main()