security/nss/mach

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 ##########################################################################
   7 #
   8 # This is a collection of helper tools to get stuff done in NSS.
   9 #
  10
  11 import sys
  12 import argparse
  13 import fnmatch
  14 import io
  15 import subprocess
  16 import os
  17 import platform
  18 import shutil
  19 import tarfile
  20 import tempfile
  21
  22 from hashlib import sha256
  23
  24 DEVNULL = open(os.devnull, 'wb')
  25 cwd = os.path.dirname(os.path.abspath(__file__))
  26
  27 def run_tests(test, cycles="standard", env={}, silent=False):
  28     domsuf = os.getenv('DOMSUF', "localdomain")
  29     host = os.getenv('HOST', "localhost")
  30     env = env.copy()
  31     env.update({
  32         "NSS_TESTS": test,
  33         "NSS_CYCLES": cycles,
  34         "DOMSUF": domsuf,
  35         "HOST": host
  36     })
  37     os_env = os.environ
  38     os_env.update(env)
  39     command = cwd + "/tests/all.sh"
  40     stdout = stderr = DEVNULL if silent else None
  41     subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr)
  42
  43
  44 class cfAction(argparse.Action):
  45     docker_command = None
  46     restorecon = None
  47
  48     def __call__(self, parser, args, values, option_string=None):
  49         self.setDockerCommand(args)
  50
  51         if values:
  52             files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values]
  53         else:
  54             files = self.modifiedFiles()
  55
  56         # First check if we can run docker.
  57         try:
  58             with open(os.devnull, "w") as f:
  59                 subprocess.check_call(
  60                     self.docker_command + ["images"], stdout=f)
  61         except:
  62             self.docker_command = None
  63
  64         if self.docker_command is None:
  65             print("warning: running clang-format directly, which isn't guaranteed to be correct")
  66             command = [cwd + "/automation/clang-format/run_clang_format.sh"] + files
  67             repr(command)
  68             subprocess.call(command)
  69             return
  70
  71         files = [os.path.join('/home/worker/nss', x) for x in files]
  72         docker_image = 'clang-format-service:latest'
  73         cf_docker_folder = cwd + "/automation/clang-format"
  74
  75         # Build the image if necessary.
  76         if self.filesChanged(cf_docker_folder):
  77             self.buildImage(docker_image, cf_docker_folder)
  78
  79         # Check if we have the docker image.
  80         try:
  81             command = self.docker_command + [
  82                 "image", "inspect", "clang-format-service:latest"
  83             ]
  84             with open(os.devnull, "w") as f:
  85                 subprocess.check_call(command, stdout=f)
  86         except:
  87             print("I have to build the docker image first.")
  88             self.buildImage(docker_image, cf_docker_folder)
  89
  90         command = self.docker_command + [
  91             'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image
  92         ]
  93         # The clang format script returns 1 if something's to do. We don't
  94         # care.
  95         subprocess.call(command + files)
  96         if self.restorecon is not None:
  97             subprocess.call([self.restorecon, '-R', cwd])
  98
  99     def filesChanged(self, path):
 100         hash = sha256()
 101         for dirname, dirnames, files in os.walk(path):
 102             for file in files:
 103                 with open(os.path.join(dirname, file), "rb") as f:
 104                     hash.update(f.read())
 105         chk_file = cwd + "/.chk"
 106         old_chk = ""
 107         new_chk = hash.hexdigest()
 108         if os.path.exists(chk_file):
 109             with open(chk_file) as f:
 110                 old_chk = f.readline()
 111         if old_chk != new_chk:
 112             with open(chk_file, "w+") as f:
 113                 f.write(new_chk)
 114             return True
 115         return False
 116
 117     def buildImage(self, docker_image, cf_docker_folder):
 118         command = self.docker_command + [
 119             "build", "-t", docker_image, cf_docker_folder
 120         ]
 121         subprocess.check_call(command)
 122         return
 123
 124     def setDockerCommand(self, args):
 125         from distutils.spawn import find_executable
 126         if platform.system() == "Linux":
 127             self.restorecon = find_executable("restorecon")
 128         dcmd = find_executable("docker")
 129         if dcmd is not None:
 130             self.docker_command = [dcmd]
 131             if not args.noroot:
 132                 self.docker_command = ["sudo"] + self.docker_command
 133         else:
 134             self.docker_command = None
 135
 136     def modifiedFiles(self):
 137         files = []
 138         if os.path.exists(os.path.join(cwd, '.hg')):
 139             st = subprocess.Popen(['hg', 'status', '-m', '-a'],
 140                                   cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True)
 141             for line in iter(st.stdout.readline, ''):
 142                 files += [line[2:].rstrip()]
 143         elif os.path.exists(os.path.join(cwd, '.git')):
 144             st = subprocess.Popen(['git', 'status', '--porcelain'],
 145                                   cwd=cwd, stdout=subprocess.PIPE)
 146             for line in iter(st.stdout.readline, ''):
 147                 if line[1] == 'M' or line[1] != 'D' and \
 148                         (line[0] == 'M' or line[0] == 'A' or
 149                          line[0] == 'C' or line[0] == 'U'):
 150                     files += [line[3:].rstrip()]
 151                 elif line[0] == 'R':
 152                     files += [line[line.index(' -> ', beg=4) + 4:]]
 153         else:
 154             print('Warning: neither mercurial nor git detected!')
 155
 156         def isFormatted(x):
 157             return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h'
 158         return [x for x in files if isFormatted(x)]
 159
 160
 161 class buildAction(argparse.Action):
 162
 163     def __call__(self, parser, args, values, option_string=None):
 164         subprocess.check_call([cwd + "/build.sh"] + values)
 165
 166
 167 class testAction(argparse.Action):
 168
 169     def __call__(self, parser, args, values, option_string=None):
 170         run_tests(values)
 171
 172
 173 class covAction(argparse.Action):
 174
 175     def runSslGtests(self, outdir):
 176         env = {
 177             "GTESTFILTER": "*", # Prevent parallel test runs.
 178             "ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir,
 179             "NSS_DEFAULT_DB_TYPE": "sql",
 180             "NSS_DISABLE_UNLOAD": "1"
 181         }
 182
 183         run_tests("ssl_gtests", env=env, silent=True)
 184
 185     def findSanCovFile(self, outdir):
 186         for file in os.listdir(outdir):
 187             if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'):
 188                 return os.path.join(outdir, file)
 189
 190         return None
 191
 192     def __call__(self, parser, args, values, option_string=None):
 193         outdir = args.outdir
 194         print("Output directory: " + outdir)
 195
 196         print("\nBuild with coverage sanitizers...\n")
 197         sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp"
 198         subprocess.check_call([
 199             os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", "--enable-legacy-db",
 200             "--sancov=" + sancov_args
 201         ])
 202
 203         print("\nRun ssl_gtests to get a coverage report...")
 204         self.runSslGtests(outdir)
 205         print("Done.")
 206
 207         sancov_file = self.findSanCovFile(outdir)
 208         if not sancov_file:
 209             print("Couldn't find .sancov file.")
 210             sys.exit(1)
 211
 212         symcov_file = os.path.join(outdir, "ssl_gtest.symcov")
 213         out = open(symcov_file, 'wb')
 214         # Don't exit immediately on error
 215         symbol_retcode = subprocess.call([
 216             "sancov",
 217             "-blacklist=" + os.path.join(cwd, ".sancov-blacklist"),
 218             "-symbolize", sancov_file,
 219             os.path.join(cwd, "../dist/Debug/bin/ssl_gtest")
 220         ], stdout=out)
 221         out.close()
 222
 223         print("\nCopying ssl_gtests to artifacts...")
 224         shutil.copyfile(os.path.join(cwd, "../dist/Debug/bin/ssl_gtest"),
 225                         os.path.join(outdir, "ssl_gtest"))
 226
 227         print("\nCoverage report: " + symcov_file)
 228         if symbol_retcode > 0:
 229             print("sancov failed to symbolize with return code {}".format(symbol_retcode))
 230         sys.exit(symbol_retcode)
 231
 232 class commandsAction(argparse.Action):
 233     commands = []
 234
 235     def __call__(self, parser, args, values, option_string=None):
 236         for c in commandsAction.commands:
 237             print(c)
 238
 239 def parse_arguments():
 240     parser = argparse.ArgumentParser(
 241         description='NSS helper script. ' +
 242         'Make sure to separate sub-command arguments with --.')
 243     subparsers = parser.add_subparsers()
 244
 245     parser_build = subparsers.add_parser(
 246         'build', help='All arguments are passed to build.sh')
 247     parser_build.add_argument(
 248         'build_args', nargs='*', help="build arguments", action=buildAction)
 249
 250     parser_cf = subparsers.add_parser(
 251         'clang-format',
 252         help="""
 253         Run clang-format.
 254
 255         By default this runs against any files that you have modified.  If
 256         there are no modified files, it checks everything.
 257         """)
 258     parser_cf.add_argument(
 259         '--noroot',
 260         help='On linux, suppress the use of \'sudo\' for running docker.',
 261         action='store_true')
 262     parser_cf.add_argument(
 263         '<file/dir>',
 264         nargs='*',
 265         help="Specify files or directories to run clang-format on",
 266         action=cfAction)
 267
 268     parser_test = subparsers.add_parser(
 269         'tests', help='Run tests through tests/all.sh.')
 270     tests = [
 271         "cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips",
 272         "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec",
 273         "gtests", "ssl_gtests", "bogo", "interop", "policy"
 274     ]
 275     parser_test.add_argument(
 276         'test', choices=tests, help="Available tests", action=testAction)
 277
 278     parser_cov = subparsers.add_parser(
 279         'coverage', help='Generate coverage report')
 280     cov_modules = ["ssl_gtests"]
 281     parser_cov.add_argument(
 282         '--outdir', help='Output directory for coverage report data.',
 283         default=tempfile.mkdtemp())
 284     parser_cov.add_argument(
 285         'module', choices=cov_modules, help="Available coverage modules",
 286         action=covAction)
 287
 288     parser_commands = subparsers.add_parser(
 289         'mach-completion',
 290         help="list commands")
 291     parser_commands.add_argument(
 292         'mach-completion',
 293         nargs='*',
 294         action=commandsAction)
 295
 296     commandsAction.commands = [c for c in subparsers.choices]
 297     return parser.parse_args()
 298
 299
 300 def main():
 301     parse_arguments()
 302
 303
 304 if __name__ == '__main__':
 305     main()