testing/mozbase/setup_development.py

   1 #!/usr/bin/env python
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5 # You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 """
   8 Setup mozbase packages for development.
   9
  10 Packages may be specified as command line arguments.
  11 If no arguments are given, install all packages.
  12
  13 See https://wiki.mozilla.org/Auto-tools/Projects/Mozbase
  14 """
  15
  16 import os
  17 import subprocess
  18 import sys
  19 from optparse import OptionParser
  20 from subprocess import PIPE
  21
  22 try:
  23     from subprocess import check_call as call
  24 except ImportError:
  25     from subprocess import call
  26
  27
  28 # directory containing this file
  29 here = os.path.dirname(os.path.abspath(__file__))
  30
  31 # all python packages
  32 mozbase_packages = [
  33     i for i in os.listdir(here) if os.path.exists(os.path.join(here, i, "setup.py"))
  34 ]
  35
  36 # testing: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Tests
  37 test_packages = ["mock"]
  38
  39 # documentation: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Documentation
  40 extra_packages = ["sphinx"]
  41
  42
  43 def cycle_check(order, dependencies):
  44     """ensure no cyclic dependencies"""
  45     order_dict = dict([(j, i) for i, j in enumerate(order)])
  46     for package, deps in dependencies.items():
  47         index = order_dict[package]
  48         for d in deps:
  49             assert index > order_dict[d], "Cyclic dependencies detected"
  50
  51
  52 def info(directory):
  53     "get the package setup.py information"
  54
  55     assert os.path.exists(os.path.join(directory, "setup.py"))
  56
  57     # setup the egg info
  58     try:
  59         call([sys.executable, "setup.py", "egg_info"], cwd=directory, stdout=PIPE)
  60     except subprocess.CalledProcessError:
  61         print("Error running setup.py in %s" % directory)
  62         raise
  63
  64     # get the .egg-info directory
  65     egg_info = [entry for entry in os.listdir(directory) if entry.endswith(".egg-info")]
  66     assert len(egg_info) == 1, "Expected one .egg-info directory in %s, got: %s" % (
  67         directory,
  68         egg_info,
  69     )
  70     egg_info = os.path.join(directory, egg_info[0])
  71     assert os.path.isdir(egg_info), "%s is not a directory" % egg_info
  72
  73     # read the package information
  74     pkg_info = os.path.join(egg_info, "PKG-INFO")
  75     info_dict = {}
  76     for line in open(pkg_info).readlines():
  77         if not line or line[0].isspace():
  78             continue  # XXX neglects description
  79         assert ":" in line
  80         key, value = [i.strip() for i in line.split(":", 1)]
  81         info_dict[key] = value
  82
  83     return info_dict
  84
  85
  86 def get_dependencies(directory):
  87     "returns the package name and dependencies given a package directory"
  88
  89     # get the package metadata
  90     info_dict = info(directory)
  91
  92     # get the .egg-info directory
  93     egg_info = [
  94         entry for entry in os.listdir(directory) if entry.endswith(".egg-info")
  95     ][0]
  96
  97     # read the dependencies
  98     requires = os.path.join(directory, egg_info, "requires.txt")
  99     dependencies = []
 100     if os.path.exists(requires):
 101         for line in open(requires):
 102             line = line.strip()
 103             # in requires.txt file, a dependency is a non empty line
 104             # Also lines like [device] are sections to mark optional
 105             # dependencies, we don't want those sections.
 106             if line and not (line.startswith("[") and line.endswith("]")):
 107                 dependencies.append(line)
 108
 109     # return the information
 110     return info_dict["Name"], dependencies
 111
 112
 113 def dependency_info(dep):
 114     "return dictionary of dependency information from a dependency string"
 115     retval = dict(Name=None, Type=None, Version=None)
 116     for joiner in ("==", "<=", ">="):
 117         if joiner in dep:
 118             retval["Type"] = joiner
 119             name, version = [i.strip() for i in dep.split(joiner, 1)]
 120             retval["Name"] = name
 121             retval["Version"] = version
 122             break
 123     else:
 124         retval["Name"] = dep.strip()
 125     return retval
 126
 127
 128 def unroll_dependencies(dependencies):
 129     """
 130     unroll a set of dependencies to a flat list
 131
 132     dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']),
 133                     'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']),
 134                     'packageC': set(['packageE']),
 135                     'packageE': set(['packageF', 'packageG']),
 136                     'packageF': set(['packageG']),
 137                     'packageX': set(['packageA', 'packageG'])}
 138     """
 139
 140     order = []
 141
 142     # flatten all
 143     packages = set(dependencies.keys())
 144     for deps in dependencies.values():
 145         packages.update(deps)
 146
 147     while len(order) != len(packages):
 148         for package in packages.difference(order):
 149             if set(dependencies.get(package, set())).issubset(order):
 150                 order.append(package)
 151                 break
 152         else:
 153             raise AssertionError("Cyclic dependencies detected")
 154
 155     cycle_check(order, dependencies)  # sanity check
 156
 157     return order
 158
 159
 160 def main(args=sys.argv[1:]):
 161     # parse command line options
 162     usage = "%prog [options] [package] [package] [...]"
 163     parser = OptionParser(usage=usage, description=__doc__)
 164     parser.add_option(
 165         "-d",
 166         "--dependencies",
 167         dest="list_dependencies",
 168         action="store_true",
 169         default=False,
 170         help="list dependencies for the packages",
 171     )
 172     parser.add_option(
 173         "--list", action="store_true", default=False, help="list what will be installed"
 174     )
 175     parser.add_option(
 176         "--extra",
 177         "--install-extra-packages",
 178         action="store_true",
 179         default=False,
 180         help="installs extra supporting packages as well as core mozbase ones",
 181     )
 182     options, packages = parser.parse_args(args)
 183
 184     if not packages:
 185         # install all packages
 186         packages = sorted(mozbase_packages)
 187
 188     # ensure specified packages are in the list
 189     assert set(packages).issubset(
 190         mozbase_packages
 191     ), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages)
 192
 193     if options.list_dependencies:
 194         # list the package dependencies
 195         for package in packages:
 196             print("%s: %s" % get_dependencies(os.path.join(here, package)))
 197         parser.exit()
 198
 199     # gather dependencies
 200     # TODO: version conflict checking
 201     deps = {}
 202     alldeps = {}
 203     mapping = {}  # mapping from subdir name to package name
 204     # core dependencies
 205     for package in packages:
 206         key, value = get_dependencies(os.path.join(here, package))
 207         deps[key] = [dependency_info(dep)["Name"] for dep in value]
 208         mapping[package] = key
 209
 210         # keep track of all dependencies for non-mozbase packages
 211         for dep in value:
 212             alldeps[dependency_info(dep)["Name"]] = "".join(dep.split())
 213
 214     # indirect dependencies
 215     flag = True
 216     while flag:
 217         flag = False
 218         for value in deps.values():
 219             for dep in value:
 220                 if dep in mozbase_packages and dep not in deps:
 221                     key, value = get_dependencies(os.path.join(here, dep))
 222                     deps[key] = [dep for dep in value]
 223
 224                     for dep in value:
 225                         alldeps[dep] = "".join(dep.split())
 226                     mapping[package] = key
 227                     flag = True
 228                     break
 229             if flag:
 230                 break
 231
 232     # get the remaining names for the mapping
 233     for package in mozbase_packages:
 234         if package in mapping:
 235             continue
 236         key, value = get_dependencies(os.path.join(here, package))
 237         mapping[package] = key
 238
 239     # unroll dependencies
 240     unrolled = unroll_dependencies(deps)
 241
 242     # make a reverse mapping: package name -> subdirectory
 243     reverse_mapping = dict([(j, i) for i, j in mapping.items()])
 244
 245     # we only care about dependencies in mozbase
 246     unrolled = [package for package in unrolled if package in reverse_mapping]
 247
 248     if options.list:
 249         # list what will be installed
 250         for package in unrolled:
 251             print(package)
 252         parser.exit()
 253
 254     # set up the packages for development
 255     for package in unrolled:
 256         call(
 257             [sys.executable, "setup.py", "develop", "--no-deps"],
 258             cwd=os.path.join(here, reverse_mapping[package]),
 259         )
 260
 261     # add the directory of sys.executable to path to aid the correct
 262     # `easy_install` getting called
 263     # https://bugzilla.mozilla.org/show_bug.cgi?id=893878
 264     os.environ["PATH"] = "%s%s%s" % (
 265         os.path.dirname(os.path.abspath(sys.executable)),
 266         os.path.pathsep,
 267         os.environ.get("PATH", "").strip(os.path.pathsep),
 268     )
 269
 270     # install non-mozbase dependencies
 271     # these need to be installed separately and the --no-deps flag
 272     # subsequently used due to a bug in setuptools; see
 273     # https://bugzilla.mozilla.org/show_bug.cgi?id=759836
 274     pypi_deps = dict([(i, j) for i, j in alldeps.items() if i not in unrolled])
 275     for package, version in pypi_deps.items():
 276         # easy_install should be available since we rely on setuptools
 277         call(["easy_install", version])
 278
 279     # install packages required for unit testing
 280     for package in test_packages:
 281         call(["easy_install", package])
 282
 283     # install extra non-mozbase packages if desired
 284     if options.extra:
 285         for package in extra_packages:
 286             call(["easy_install", package])
 287
 288
 289 if __name__ == "__main__":
 290     main()