Bug 1886946: Remove incorrect assertion that buffer is not-pinned. r=sfink
[gecko.git] / security / nss / mach
blob3d72baa2ea28c11ee93fa67b3566ace139dfaef4
1 #!/usr/bin/env python
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 ##########################################################################
8 # This is a collection of helper tools to get stuff done in NSS.
11 import sys
12 import argparse
13 import fnmatch
14 import io
15 import subprocess
16 import os
17 import platform
18 import shutil
19 import tarfile
20 import tempfile
22 from hashlib import sha256
24 DEVNULL = open(os.devnull, 'wb')
25 cwd = os.path.dirname(os.path.abspath(__file__))
27 def run_tests(test, cycles="standard", env={}, silent=False):
28 domsuf = os.getenv('DOMSUF', "localdomain")
29 host = os.getenv('HOST', "localhost")
30 env = env.copy()
31 env.update({
32 "NSS_TESTS": test,
33 "NSS_CYCLES": cycles,
34 "DOMSUF": domsuf,
35 "HOST": host
37 os_env = os.environ
38 os_env.update(env)
39 command = cwd + "/tests/all.sh"
40 stdout = stderr = DEVNULL if silent else None
41 subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr)
44 class cfAction(argparse.Action):
45 docker_command = None
46 restorecon = None
48 def __call__(self, parser, args, values, option_string=None):
49 self.setDockerCommand(args)
51 if values:
52 files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values]
53 else:
54 files = self.modifiedFiles()
56 # First check if we can run docker.
57 try:
58 with open(os.devnull, "w") as f:
59 subprocess.check_call(
60 self.docker_command + ["images"], stdout=f)
61 except:
62 self.docker_command = None
64 if self.docker_command is None:
65 print("warning: running clang-format directly, which isn't guaranteed to be correct")
66 command = [cwd + "/automation/clang-format/run_clang_format.sh"] + files
67 repr(command)
68 subprocess.call(command)
69 return
71 files = [os.path.join('/home/worker/nss', x) for x in files]
72 docker_image = 'clang-format-service:latest'
73 cf_docker_folder = cwd + "/automation/clang-format"
75 # Build the image if necessary.
76 if self.filesChanged(cf_docker_folder):
77 self.buildImage(docker_image, cf_docker_folder)
79 # Check if we have the docker image.
80 try:
81 command = self.docker_command + [
82 "image", "inspect", "clang-format-service:latest"
84 with open(os.devnull, "w") as f:
85 subprocess.check_call(command, stdout=f)
86 except:
87 print("I have to build the docker image first.")
88 self.buildImage(docker_image, cf_docker_folder)
90 command = self.docker_command + [
91 'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image
93 # The clang format script returns 1 if something's to do. We don't
94 # care.
95 subprocess.call(command + files)
96 if self.restorecon is not None:
97 subprocess.call([self.restorecon, '-R', cwd])
99 def filesChanged(self, path):
100 hash = sha256()
101 for dirname, dirnames, files in os.walk(path):
102 for file in files:
103 with open(os.path.join(dirname, file), "rb") as f:
104 hash.update(f.read())
105 chk_file = cwd + "/.chk"
106 old_chk = ""
107 new_chk = hash.hexdigest()
108 if os.path.exists(chk_file):
109 with open(chk_file) as f:
110 old_chk = f.readline()
111 if old_chk != new_chk:
112 with open(chk_file, "w+") as f:
113 f.write(new_chk)
114 return True
115 return False
117 def buildImage(self, docker_image, cf_docker_folder):
118 command = self.docker_command + [
119 "build", "-t", docker_image, cf_docker_folder
121 subprocess.check_call(command)
122 return
124 def setDockerCommand(self, args):
125 if platform.system() == "Linux":
126 self.restorecon = shutil.which("restorecon")
127 dcmd = shutil.which("docker")
128 if dcmd is not None:
129 self.docker_command = [dcmd]
130 if not args.noroot:
131 self.docker_command = ["sudo"] + self.docker_command
132 else:
133 self.docker_command = None
135 def modifiedFiles(self):
136 files = []
137 if os.path.exists(os.path.join(cwd, '.hg')):
138 st = subprocess.Popen(['hg', 'status', '-m', '-a'],
139 cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True)
140 for line in iter(st.stdout.readline, ''):
141 files += [line[2:].rstrip()]
142 elif os.path.exists(os.path.join(cwd, '.git')):
143 st = subprocess.Popen(['git', 'status', '--porcelain'],
144 cwd=cwd, stdout=subprocess.PIPE)
145 for line in iter(st.stdout.readline, ''):
146 if line[1] == 'M' or line[1] != 'D' and \
147 (line[0] == 'M' or line[0] == 'A' or
148 line[0] == 'C' or line[0] == 'U'):
149 files += [line[3:].rstrip()]
150 elif line[0] == 'R':
151 files += [line[line.index(' -> ', beg=4) + 4:]]
152 else:
153 print('Warning: neither mercurial nor git detected!')
155 def isFormatted(x):
156 return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h'
157 return [x for x in files if isFormatted(x)]
160 class buildAction(argparse.Action):
162 def __call__(self, parser, args, values, option_string=None):
163 subprocess.check_call([cwd + "/build.sh"] + values)
166 class testAction(argparse.Action):
168 def __call__(self, parser, args, values, option_string=None):
169 run_tests(values)
172 class covAction(argparse.Action):
174 def runSslGtests(self, outdir):
175 env = {
176 "GTESTFILTER": "*", # Prevent parallel test runs.
177 "ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir,
178 "NSS_DEFAULT_DB_TYPE": "sql",
179 "NSS_DISABLE_UNLOAD": "1"
182 run_tests("ssl_gtests", env=env, silent=True)
184 def findSanCovFile(self, outdir):
185 for file in os.listdir(outdir):
186 if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'):
187 return os.path.join(outdir, file)
189 return None
191 def __call__(self, parser, args, values, option_string=None):
192 outdir = args.outdir
193 print("Output directory: " + outdir)
195 print("\nBuild with coverage sanitizers...\n")
196 sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp"
197 subprocess.check_call([
198 os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", "--enable-legacy-db",
199 "--sancov=" + sancov_args
202 print("\nRun ssl_gtests to get a coverage report...")
203 self.runSslGtests(outdir)
204 print("Done.")
206 sancov_file = self.findSanCovFile(outdir)
207 if not sancov_file:
208 print("Couldn't find .sancov file.")
209 sys.exit(1)
211 symcov_file = os.path.join(outdir, "ssl_gtest.symcov")
212 out = open(symcov_file, 'wb')
213 # Don't exit immediately on error
214 symbol_retcode = subprocess.call([
215 "sancov",
216 "-blacklist=" + os.path.join(cwd, ".sancov-blacklist"),
217 "-symbolize", sancov_file,
218 os.path.join(cwd, "../dist/Debug/bin/ssl_gtest")
219 ], stdout=out)
220 out.close()
222 print("\nCopying ssl_gtests to artifacts...")
223 shutil.copyfile(os.path.join(cwd, "../dist/Debug/bin/ssl_gtest"),
224 os.path.join(outdir, "ssl_gtest"))
226 print("\nCoverage report: " + symcov_file)
227 if symbol_retcode > 0:
228 print("sancov failed to symbolize with return code {}".format(symbol_retcode))
229 sys.exit(symbol_retcode)
231 class commandsAction(argparse.Action):
232 commands = []
234 def __call__(self, parser, args, values, option_string=None):
235 for c in commandsAction.commands:
236 print(c)
238 def parse_arguments():
239 parser = argparse.ArgumentParser(
240 description='NSS helper script. ' +
241 'Make sure to separate sub-command arguments with --.')
242 subparsers = parser.add_subparsers()
244 parser_build = subparsers.add_parser(
245 'build', help='All arguments are passed to build.sh')
246 parser_build.add_argument(
247 'build_args', nargs='*', help="build arguments", action=buildAction)
249 parser_cf = subparsers.add_parser(
250 'clang-format',
251 help="""
252 Run clang-format.
254 By default this runs against any files that you have modified. If
255 there are no modified files, it checks everything.
256 """)
257 parser_cf.add_argument(
258 '--noroot',
259 help='On linux, suppress the use of \'sudo\' for running docker.',
260 action='store_true')
261 parser_cf.add_argument(
262 '<file/dir>',
263 nargs='*',
264 help="Specify files or directories to run clang-format on",
265 action=cfAction)
267 parser_test = subparsers.add_parser(
268 'tests', help='Run tests through tests/all.sh.')
269 tests = [
270 "cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips",
271 "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec",
272 "gtests", "ssl_gtests", "bogo", "interop", "policy"
274 parser_test.add_argument(
275 'test', choices=tests, help="Available tests", action=testAction)
277 parser_cov = subparsers.add_parser(
278 'coverage', help='Generate coverage report')
279 cov_modules = ["ssl_gtests"]
280 parser_cov.add_argument(
281 '--outdir', help='Output directory for coverage report data.',
282 default=tempfile.mkdtemp())
283 parser_cov.add_argument(
284 'module', choices=cov_modules, help="Available coverage modules",
285 action=covAction)
287 parser_commands = subparsers.add_parser(
288 'mach-completion',
289 help="list commands")
290 parser_commands.add_argument(
291 'mach-completion',
292 nargs='*',
293 action=commandsAction)
295 commandsAction.commands = [c for c in subparsers.choices]
296 return parser.parse_args()
299 def main():
300 parse_arguments()
303 if __name__ == '__main__':
304 main()