security/nss/mach

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 ##########################################################################
   7 #
   8 # This is a collection of helper tools to get stuff done in NSS.
   9 #
  10
  11 import sys
  12 import argparse
  13 import fnmatch
  14 import io
  15 import subprocess
  16 import os
  17 import platform
  18 import shutil
  19 import tarfile
  20 import tempfile
  21
  22 from hashlib import sha256
  23
  24 DEVNULL = open(os.devnull, 'wb')
  25 cwd = os.path.dirname(os.path.abspath(__file__))
  26
  27 def run_tests(test, cycles="standard", env={}, silent=False):
  28     domsuf = os.getenv('DOMSUF', "localdomain")
  29     host = os.getenv('HOST', "localhost")
  30     env = env.copy()
  31     env.update({
  32         "NSS_TESTS": test,
  33         "NSS_CYCLES": cycles,
  34         "DOMSUF": domsuf,
  35         "HOST": host
  36     })
  37     os_env = os.environ
  38     os_env.update(env)
  39     command = cwd + "/tests/all.sh"
  40     stdout = stderr = DEVNULL if silent else None
  41     subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr)
  42
  43
  44 class cfAction(argparse.Action):
  45     docker_command = None
  46     restorecon = None
  47
  48     def __call__(self, parser, args, values, option_string=None):
  49         self.setDockerCommand(args)
  50
  51         if values:
  52             files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values]
  53         else:
  54             files = self.modifiedFiles()
  55
  56         # First check if we can run docker.
  57         try:
  58             with open(os.devnull, "w") as f:
  59                 subprocess.check_call(
  60                     self.docker_command + ["images"], stdout=f)
  61         except:
  62             self.docker_command = None
  63
  64         if self.docker_command is None:
  65             print("warning: running clang-format directly, which isn't guaranteed to be correct")
  66             command = [cwd + "/automation/clang-format/run_clang_format.sh"] + files
  67             repr(command)
  68             subprocess.call(command)
  69             return
  70
  71         files = [os.path.join('/home/worker/nss', x) for x in files]
  72         docker_image = 'clang-format-service:latest'
  73         cf_docker_folder = cwd + "/automation/clang-format"
  74
  75         # Build the image if necessary.
  76         if self.filesChanged(cf_docker_folder):
  77             self.buildImage(docker_image, cf_docker_folder)
  78
  79         # Check if we have the docker image.
  80         try:
  81             command = self.docker_command + [
  82                 "image", "inspect", "clang-format-service:latest"
  83             ]
  84             with open(os.devnull, "w") as f:
  85                 subprocess.check_call(command, stdout=f)
  86         except:
  87             print("I have to build the docker image first.")
  88             self.buildImage(docker_image, cf_docker_folder)
  89
  90         command = self.docker_command + [
  91             'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image
  92         ]
  93         # The clang format script returns 1 if something's to do. We don't
  94         # care.
  95         subprocess.call(command + files)
  96         if self.restorecon is not None:
  97             subprocess.call([self.restorecon, '-R', cwd])
  98
  99     def filesChanged(self, path):
 100         hash = sha256()
 101         for dirname, dirnames, files in os.walk(path):
 102             for file in files:
 103                 with open(os.path.join(dirname, file), "rb") as f:
 104                     hash.update(f.read())
 105         chk_file = cwd + "/.chk"
 106         old_chk = ""
 107         new_chk = hash.hexdigest()
 108         if os.path.exists(chk_file):
 109             with open(chk_file) as f:
 110                 old_chk = f.readline()
 111         if old_chk != new_chk:
 112             with open(chk_file, "w+") as f:
 113                 f.write(new_chk)
 114             return True
 115         return False
 116
 117     def buildImage(self, docker_image, cf_docker_folder):
 118         command = self.docker_command + [
 119             "build", "-t", docker_image, cf_docker_folder
 120         ]
 121         subprocess.check_call(command)
 122         return
 123
 124     def setDockerCommand(self, args):
 125         if platform.system() == "Linux":
 126             self.restorecon = shutil.which("restorecon")
 127         dcmd = shutil.which("docker")
 128         if dcmd is not None:
 129             self.docker_command = [dcmd]
 130             if not args.noroot:
 131                 self.docker_command = ["sudo"] + self.docker_command
 132         else:
 133             self.docker_command = None
 134
 135     def modifiedFiles(self):
 136         files = []
 137         if os.path.exists(os.path.join(cwd, '.hg')):
 138             st = subprocess.Popen(['hg', 'status', '-m', '-a'],
 139                                   cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True)
 140             for line in iter(st.stdout.readline, ''):
 141                 files += [line[2:].rstrip()]
 142         elif os.path.exists(os.path.join(cwd, '.git')):
 143             st = subprocess.Popen(['git', 'status', '--porcelain'],
 144                                   cwd=cwd, stdout=subprocess.PIPE)
 145             for line in iter(st.stdout.readline, ''):
 146                 if line[1] == 'M' or line[1] != 'D' and \
 147                         (line[0] == 'M' or line[0] == 'A' or
 148                          line[0] == 'C' or line[0] == 'U'):
 149                     files += [line[3:].rstrip()]
 150                 elif line[0] == 'R':
 151                     files += [line[line.index(' -> ', beg=4) + 4:]]
 152         else:
 153             print('Warning: neither mercurial nor git detected!')
 154
 155         def isFormatted(x):
 156             return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h'
 157         return [x for x in files if isFormatted(x)]
 158
 159
 160 class buildAction(argparse.Action):
 161
 162     def __call__(self, parser, args, values, option_string=None):
 163         subprocess.check_call([cwd + "/build.sh"] + values)
 164
 165
 166 class testAction(argparse.Action):
 167
 168     def __call__(self, parser, args, values, option_string=None):
 169         run_tests(values)
 170
 171
 172 class covAction(argparse.Action):
 173
 174     def runSslGtests(self, outdir):
 175         env = {
 176             "GTESTFILTER": "*", # Prevent parallel test runs.
 177             "ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir,
 178             "NSS_DEFAULT_DB_TYPE": "sql",
 179             "NSS_DISABLE_UNLOAD": "1"
 180         }
 181
 182         run_tests("ssl_gtests", env=env, silent=True)
 183
 184     def findSanCovFile(self, outdir):
 185         for file in os.listdir(outdir):
 186             if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'):
 187                 return os.path.join(outdir, file)
 188
 189         return None
 190
 191     def __call__(self, parser, args, values, option_string=None):
 192         outdir = args.outdir
 193         print("Output directory: " + outdir)
 194
 195         print("\nBuild with coverage sanitizers...\n")
 196         sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp"
 197         subprocess.check_call([
 198             os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", "--enable-legacy-db",
 199             "--sancov=" + sancov_args
 200         ])
 201
 202         print("\nRun ssl_gtests to get a coverage report...")
 203         self.runSslGtests(outdir)
 204         print("Done.")
 205
 206         sancov_file = self.findSanCovFile(outdir)
 207         if not sancov_file:
 208             print("Couldn't find .sancov file.")
 209             sys.exit(1)
 210
 211         symcov_file = os.path.join(outdir, "ssl_gtest.symcov")
 212         out = open(symcov_file, 'wb')
 213         # Don't exit immediately on error
 214         symbol_retcode = subprocess.call([
 215             "sancov",
 216             "-blacklist=" + os.path.join(cwd, ".sancov-blacklist"),
 217             "-symbolize", sancov_file,
 218             os.path.join(cwd, "../dist/Debug/bin/ssl_gtest")
 219         ], stdout=out)
 220         out.close()
 221
 222         print("\nCopying ssl_gtests to artifacts...")
 223         shutil.copyfile(os.path.join(cwd, "../dist/Debug/bin/ssl_gtest"),
 224                         os.path.join(outdir, "ssl_gtest"))
 225
 226         print("\nCoverage report: " + symcov_file)
 227         if symbol_retcode > 0:
 228             print("sancov failed to symbolize with return code {}".format(symbol_retcode))
 229         sys.exit(symbol_retcode)
 230
 231 class commandsAction(argparse.Action):
 232     commands = []
 233
 234     def __call__(self, parser, args, values, option_string=None):
 235         for c in commandsAction.commands:
 236             print(c)
 237
 238 def parse_arguments():
 239     parser = argparse.ArgumentParser(
 240         description='NSS helper script. ' +
 241         'Make sure to separate sub-command arguments with --.')
 242     subparsers = parser.add_subparsers()
 243
 244     parser_build = subparsers.add_parser(
 245         'build', help='All arguments are passed to build.sh')
 246     parser_build.add_argument(
 247         'build_args', nargs='*', help="build arguments", action=buildAction)
 248
 249     parser_cf = subparsers.add_parser(
 250         'clang-format',
 251         help="""
 252         Run clang-format.
 253
 254         By default this runs against any files that you have modified.  If
 255         there are no modified files, it checks everything.
 256         """)
 257     parser_cf.add_argument(
 258         '--noroot',
 259         help='On linux, suppress the use of \'sudo\' for running docker.',
 260         action='store_true')
 261     parser_cf.add_argument(
 262         '<file/dir>',
 263         nargs='*',
 264         help="Specify files or directories to run clang-format on",
 265         action=cfAction)
 266
 267     parser_test = subparsers.add_parser(
 268         'tests', help='Run tests through tests/all.sh.')
 269     tests = [
 270         "cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips",
 271         "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec",
 272         "gtests", "ssl_gtests", "bogo", "interop", "policy"
 273     ]
 274     parser_test.add_argument(
 275         'test', choices=tests, help="Available tests", action=testAction)
 276
 277     parser_cov = subparsers.add_parser(
 278         'coverage', help='Generate coverage report')
 279     cov_modules = ["ssl_gtests"]
 280     parser_cov.add_argument(
 281         '--outdir', help='Output directory for coverage report data.',
 282         default=tempfile.mkdtemp())
 283     parser_cov.add_argument(
 284         'module', choices=cov_modules, help="Available coverage modules",
 285         action=covAction)
 286
 287     parser_commands = subparsers.add_parser(
 288         'mach-completion',
 289         help="list commands")
 290     parser_commands.add_argument(
 291         'mach-completion',
 292         nargs='*',
 293         action=commandsAction)
 294
 295     commandsAction.commands = [c for c in subparsers.choices]
 296     return parser.parse_args()
 297
 298
 299 def main():
 300     parse_arguments()
 301
 302
 303 if __name__ == '__main__':
 304     main()