testing/mozbase/setup_development.py

   1 #!/usr/bin/env python
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5 # You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 """
   8 Setup mozbase packages for development.
   9
  10 Packages may be specified as command line arguments.
  11 If no arguments are given, install all packages.
  12
  13 See https://wiki.mozilla.org/Auto-tools/Projects/Mozbase
  14 """
  15
  16 from __future__ import absolute_import, print_function
  17
  18 import os
  19 import subprocess
  20 import sys
  21 from optparse import OptionParser
  22 from subprocess import PIPE
  23
  24 try:
  25     from subprocess import check_call as call
  26 except ImportError:
  27     from subprocess import call
  28
  29
  30 # directory containing this file
  31 here = os.path.dirname(os.path.abspath(__file__))
  32
  33 # all python packages
  34 mozbase_packages = [
  35     i for i in os.listdir(here) if os.path.exists(os.path.join(here, i, "setup.py"))
  36 ]
  37
  38 # testing: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Tests
  39 test_packages = ["mock"]
  40
  41 # documentation: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Documentation
  42 extra_packages = ["sphinx"]
  43
  44
  45 def cycle_check(order, dependencies):
  46     """ensure no cyclic dependencies"""
  47     order_dict = dict([(j, i) for i, j in enumerate(order)])
  48     for package, deps in dependencies.items():
  49         index = order_dict[package]
  50         for d in deps:
  51             assert index > order_dict[d], "Cyclic dependencies detected"
  52
  53
  54 def info(directory):
  55     "get the package setup.py information"
  56
  57     assert os.path.exists(os.path.join(directory, "setup.py"))
  58
  59     # setup the egg info
  60     try:
  61         call([sys.executable, "setup.py", "egg_info"], cwd=directory, stdout=PIPE)
  62     except subprocess.CalledProcessError:
  63         print("Error running setup.py in %s" % directory)
  64         raise
  65
  66     # get the .egg-info directory
  67     egg_info = [entry for entry in os.listdir(directory) if entry.endswith(".egg-info")]
  68     assert len(egg_info) == 1, "Expected one .egg-info directory in %s, got: %s" % (
  69         directory,
  70         egg_info,
  71     )
  72     egg_info = os.path.join(directory, egg_info[0])
  73     assert os.path.isdir(egg_info), "%s is not a directory" % egg_info
  74
  75     # read the package information
  76     pkg_info = os.path.join(egg_info, "PKG-INFO")
  77     info_dict = {}
  78     for line in open(pkg_info).readlines():
  79         if not line or line[0].isspace():
  80             continue  # XXX neglects description
  81         assert ":" in line
  82         key, value = [i.strip() for i in line.split(":", 1)]
  83         info_dict[key] = value
  84
  85     return info_dict
  86
  87
  88 def get_dependencies(directory):
  89     "returns the package name and dependencies given a package directory"
  90
  91     # get the package metadata
  92     info_dict = info(directory)
  93
  94     # get the .egg-info directory
  95     egg_info = [
  96         entry for entry in os.listdir(directory) if entry.endswith(".egg-info")
  97     ][0]
  98
  99     # read the dependencies
 100     requires = os.path.join(directory, egg_info, "requires.txt")
 101     dependencies = []
 102     if os.path.exists(requires):
 103         for line in open(requires):
 104             line = line.strip()
 105             # in requires.txt file, a dependency is a non empty line
 106             # Also lines like [device] are sections to mark optional
 107             # dependencies, we don't want those sections.
 108             if line and not (line.startswith("[") and line.endswith("]")):
 109                 dependencies.append(line)
 110
 111     # return the information
 112     return info_dict["Name"], dependencies
 113
 114
 115 def dependency_info(dep):
 116     "return dictionary of dependency information from a dependency string"
 117     retval = dict(Name=None, Type=None, Version=None)
 118     for joiner in ("==", "<=", ">="):
 119         if joiner in dep:
 120             retval["Type"] = joiner
 121             name, version = [i.strip() for i in dep.split(joiner, 1)]
 122             retval["Name"] = name
 123             retval["Version"] = version
 124             break
 125     else:
 126         retval["Name"] = dep.strip()
 127     return retval
 128
 129
 130 def unroll_dependencies(dependencies):
 131     """
 132     unroll a set of dependencies to a flat list
 133
 134     dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']),
 135                     'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']),
 136                     'packageC': set(['packageE']),
 137                     'packageE': set(['packageF', 'packageG']),
 138                     'packageF': set(['packageG']),
 139                     'packageX': set(['packageA', 'packageG'])}
 140     """
 141
 142     order = []
 143
 144     # flatten all
 145     packages = set(dependencies.keys())
 146     for deps in dependencies.values():
 147         packages.update(deps)
 148
 149     while len(order) != len(packages):
 150
 151         for package in packages.difference(order):
 152             if set(dependencies.get(package, set())).issubset(order):
 153                 order.append(package)
 154                 break
 155         else:
 156             raise AssertionError("Cyclic dependencies detected")
 157
 158     cycle_check(order, dependencies)  # sanity check
 159
 160     return order
 161
 162
 163 def main(args=sys.argv[1:]):
 164
 165     # parse command line options
 166     usage = "%prog [options] [package] [package] [...]"
 167     parser = OptionParser(usage=usage, description=__doc__)
 168     parser.add_option(
 169         "-d",
 170         "--dependencies",
 171         dest="list_dependencies",
 172         action="store_true",
 173         default=False,
 174         help="list dependencies for the packages",
 175     )
 176     parser.add_option(
 177         "--list", action="store_true", default=False, help="list what will be installed"
 178     )
 179     parser.add_option(
 180         "--extra",
 181         "--install-extra-packages",
 182         action="store_true",
 183         default=False,
 184         help="installs extra supporting packages as well as core mozbase ones",
 185     )
 186     options, packages = parser.parse_args(args)
 187
 188     if not packages:
 189         # install all packages
 190         packages = sorted(mozbase_packages)
 191
 192     # ensure specified packages are in the list
 193     assert set(packages).issubset(
 194         mozbase_packages
 195     ), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages)
 196
 197     if options.list_dependencies:
 198         # list the package dependencies
 199         for package in packages:
 200             print("%s: %s" % get_dependencies(os.path.join(here, package)))
 201         parser.exit()
 202
 203     # gather dependencies
 204     # TODO: version conflict checking
 205     deps = {}
 206     alldeps = {}
 207     mapping = {}  # mapping from subdir name to package name
 208     # core dependencies
 209     for package in packages:
 210         key, value = get_dependencies(os.path.join(here, package))
 211         deps[key] = [dependency_info(dep)["Name"] for dep in value]
 212         mapping[package] = key
 213
 214         # keep track of all dependencies for non-mozbase packages
 215         for dep in value:
 216             alldeps[dependency_info(dep)["Name"]] = "".join(dep.split())
 217
 218     # indirect dependencies
 219     flag = True
 220     while flag:
 221         flag = False
 222         for value in deps.values():
 223             for dep in value:
 224                 if dep in mozbase_packages and dep not in deps:
 225                     key, value = get_dependencies(os.path.join(here, dep))
 226                     deps[key] = [dep for dep in value]
 227
 228                     for dep in value:
 229                         alldeps[dep] = "".join(dep.split())
 230                     mapping[package] = key
 231                     flag = True
 232                     break
 233             if flag:
 234                 break
 235
 236     # get the remaining names for the mapping
 237     for package in mozbase_packages:
 238         if package in mapping:
 239             continue
 240         key, value = get_dependencies(os.path.join(here, package))
 241         mapping[package] = key
 242
 243     # unroll dependencies
 244     unrolled = unroll_dependencies(deps)
 245
 246     # make a reverse mapping: package name -> subdirectory
 247     reverse_mapping = dict([(j, i) for i, j in mapping.items()])
 248
 249     # we only care about dependencies in mozbase
 250     unrolled = [package for package in unrolled if package in reverse_mapping]
 251
 252     if options.list:
 253         # list what will be installed
 254         for package in unrolled:
 255             print(package)
 256         parser.exit()
 257
 258     # set up the packages for development
 259     for package in unrolled:
 260         call(
 261             [sys.executable, "setup.py", "develop", "--no-deps"],
 262             cwd=os.path.join(here, reverse_mapping[package]),
 263         )
 264
 265     # add the directory of sys.executable to path to aid the correct
 266     # `easy_install` getting called
 267     # https://bugzilla.mozilla.org/show_bug.cgi?id=893878
 268     os.environ["PATH"] = "%s%s%s" % (
 269         os.path.dirname(os.path.abspath(sys.executable)),
 270         os.path.pathsep,
 271         os.environ.get("PATH", "").strip(os.path.pathsep),
 272     )
 273
 274     # install non-mozbase dependencies
 275     # these need to be installed separately and the --no-deps flag
 276     # subsequently used due to a bug in setuptools; see
 277     # https://bugzilla.mozilla.org/show_bug.cgi?id=759836
 278     pypi_deps = dict([(i, j) for i, j in alldeps.items() if i not in unrolled])
 279     for package, version in pypi_deps.items():
 280         # easy_install should be available since we rely on setuptools
 281         call(["easy_install", version])
 282
 283     # install packages required for unit testing
 284     for package in test_packages:
 285         call(["easy_install", package])
 286
 287     # install extra non-mozbase packages if desired
 288     if options.extra:
 289         for package in extra_packages:
 290             call(["easy_install", package])
 291
 292
 293 if __name__ == "__main__":
 294     main()