dom/imptests/importTestsuite.py

   1 #!/usr/bin/env python
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4 # You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 """
   7 Imports a test suite from a remote repository. Takes one argument, a file in
   8 the format described in README.
   9 Note: removes both source and destination directory before starting. Do not
  10       use with outstanding changes in either directory.
  11 """
  12
  13 from __future__ import print_function, unicode_literals
  14
  15 import os
  16 import shutil
  17 import subprocess
  18 import sys
  19
  20 import parseManifest
  21 import writeBuildFiles
  22
  23 def readManifests(iden, dirs):
  24     def parseManifestFile(iden, path):
  25         pathstr = "hg-%s/%s/MANIFEST" % (iden, path)
  26         subdirs, mochitests, reftests, _, supportfiles = parseManifest.parseManifestFile(pathstr)
  27         return subdirs, mochitests, reftests, supportfiles
  28
  29     data = []
  30     for path in dirs:
  31         subdirs, mochitests, reftests, supportfiles = parseManifestFile(iden, path)
  32         data.append({
  33           "path": path,
  34           "mochitests": mochitests,
  35           "reftests": reftests,
  36           "supportfiles": supportfiles,
  37         })
  38         data.extend(readManifests(iden, ["%s/%s" % (path, d) for d in subdirs]))
  39     return data
  40
  41
  42 def getData(confFile):
  43     """This function parses a file of the form
  44     (hg or git)|URL of remote repository|identifier for the local directory
  45     First directory of tests
  46     ...
  47     Last directory of tests"""
  48     vcs = ""
  49     url = ""
  50     iden = ""
  51     directories = []
  52     try:
  53         with open(confFile, 'r') as fp:
  54             first = True
  55             for line in fp:
  56                 if first:
  57                     vcs, url, iden = line.strip().split("|")
  58                     first = False
  59                 else:
  60                     directories.append(line.strip())
  61     finally:
  62         return vcs, url, iden, directories
  63
  64
  65 def makePathInternal(a, b):
  66     if not b:
  67         # Empty directory, i.e., the repository root.
  68         return a
  69     return "%s/%s" % (a, b)
  70
  71
  72 def makeSourcePath(a, b):
  73     """Make a path in the source (upstream) directory."""
  74     return makePathInternal("hg-%s" % a, b)
  75
  76
  77 def makeDestPath(a, b):
  78     """Make a path in the destination (mozilla-central) directory, shortening as
  79     appropriate."""
  80     def shorten(path):
  81         path = path.replace('dom-tree-accessors', 'dta')
  82         path = path.replace('document.getElementsByName', 'doc.gEBN')
  83         path = path.replace('requirements-for-implementations', 'implreq')
  84         path = path.replace('other-elements-attributes-and-apis', 'oeaaa')
  85         return path
  86
  87     return shorten(makePathInternal(a, b))
  88
  89
  90 def extractReftestFiles(reftests):
  91     """Returns the set of files referenced in the reftests argument"""
  92     files = set()
  93     for line in reftests:
  94         files.update([line[1], line[2]])
  95     return files
  96
  97
  98 def copy(dest, directories):
  99     """Copy mochitests and support files from the external HG directory to their
 100     place in mozilla-central.
 101     """
 102     print("Copying tests...")
 103     for d in directories:
 104         sourcedir = makeSourcePath(dest, d["path"])
 105         destdir = makeDestPath(dest, d["path"])
 106         os.makedirs(destdir)
 107
 108         reftestfiles = extractReftestFiles(d["reftests"])
 109
 110         for mochitest in d["mochitests"]:
 111             shutil.copy("%s/%s" % (sourcedir, mochitest), "%s/test_%s" % (destdir, mochitest))
 112         for reftest in sorted(reftestfiles):
 113             shutil.copy("%s/%s" % (sourcedir, reftest), "%s/%s" % (destdir, reftest))
 114         for support in d["supportfiles"]:
 115             shutil.copy("%s/%s" % (sourcedir, support), "%s/%s" % (destdir, support))
 116
 117 def printBuildFiles(dest, directories):
 118     """Create a mochitest.ini that all the contains tests we import.
 119     """
 120     print("Creating manifest...")
 121     all_mochitests = set()
 122     all_support = set()
 123
 124     for d in directories:
 125         path = makeDestPath(dest, d["path"])
 126
 127         all_mochitests |= set('%s/test_%s' % (d['path'], mochitest)
 128             for mochitest in d['mochitests'])
 129         all_support |= set('%s/%s' % (d['path'], p) for p in d['supportfiles'])
 130
 131         if d["reftests"]:
 132             with open(path + "/reftest.list", "w") as fh:
 133                 result = writeBuildFiles.substReftestList("importTestsuite.py",
 134                     d["reftests"])
 135                 fh.write(result)
 136
 137     manifest_path = dest + '/mochitest.ini'
 138     with open(manifest_path, 'w') as fh:
 139         result = writeBuildFiles.substManifest('importTestsuite.py',
 140             all_mochitests, all_support)
 141         fh.write(result)
 142     subprocess.check_call(["hg", "add", manifest_path])
 143
 144 def hgadd(dest, directories):
 145     """Inform hg of the files in |directories|."""
 146     print("hg addremoving...")
 147     for d in directories:
 148         subprocess.check_call(["hg", "addremove", makeDestPath(dest, d)])
 149
 150 def removeAndCloneRepo(vcs, url, dest):
 151     """Replaces the repo at dest by a fresh clone from url using vcs"""
 152     assert vcs in ('hg', 'git')
 153
 154     print("Removing %s..." % dest)
 155     subprocess.check_call(["rm", "-rf", dest])
 156
 157     print("Cloning %s to %s with %s..." % (url, dest, vcs))
 158     subprocess.check_call([vcs, "clone", url, dest])
 159
 160 def importRepo(confFile):
 161     try:
 162         vcs, url, iden, directories = getData(confFile)
 163         dest = iden
 164         hgdest = "hg-%s" % iden
 165
 166         print("Removing %s..." % dest)
 167         subprocess.check_call(["rm", "-rf", dest])
 168
 169         removeAndCloneRepo(vcs, url, hgdest)
 170
 171         data = readManifests(iden, directories)
 172         print("Going to import %s..." % [d["path"] for d in data])
 173
 174         copy(dest, data)
 175         printBuildFiles(dest, data)
 176         hgadd(dest, directories)
 177         print("Removing %s again..." % hgdest)
 178         subprocess.check_call(["rm", "-rf", hgdest])
 179     except subprocess.CalledProcessError as e:
 180         print(e.returncode)
 181     finally:
 182         print("Done")
 183
 184 if __name__ == "__main__":
 185     if len(sys.argv) != 2:
 186         print("Need one argument.")
 187     else:
 188         importRepo(sys.argv[1])
 189