3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 ##########################################################################
8 # This is a collection of helper tools to get stuff done in NSS.
22 from hashlib
import sha256
24 DEVNULL
= open(os
.devnull
, 'wb')
25 cwd
= os
.path
.dirname(os
.path
.abspath(__file__
))
27 def run_tests(test
, cycles
="standard", env
={}, silent
=False):
28 domsuf
= os
.getenv('DOMSUF', "localdomain")
29 host
= os
.getenv('HOST', "localhost")
39 command
= cwd
+ "/tests/all.sh"
40 stdout
= stderr
= DEVNULL
if silent
else None
41 subprocess
.check_call(command
, env
=os_env
, stdout
=stdout
, stderr
=stderr
)
44 class cfAction(argparse
.Action
):
48 def __call__(self
, parser
, args
, values
, option_string
=None):
49 self
.setDockerCommand(args
)
52 files
= [os
.path
.relpath(os
.path
.abspath(x
), start
=cwd
) for x
in values
]
54 files
= self
.modifiedFiles()
56 # First check if we can run docker.
58 with
open(os
.devnull
, "w") as f
:
59 subprocess
.check_call(
60 self
.docker_command
+ ["images"], stdout
=f
)
62 self
.docker_command
= None
64 if self
.docker_command
is None:
65 print("warning: running clang-format directly, which isn't guaranteed to be correct")
66 command
= [cwd
+ "/automation/clang-format/run_clang_format.sh"] + files
68 subprocess
.call(command
)
71 files
= [os
.path
.join('/home/worker/nss', x
) for x
in files
]
72 docker_image
= 'clang-format-service:latest'
73 cf_docker_folder
= cwd
+ "/automation/clang-format"
75 # Build the image if necessary.
76 if self
.filesChanged(cf_docker_folder
):
77 self
.buildImage(docker_image
, cf_docker_folder
)
79 # Check if we have the docker image.
81 command
= self
.docker_command
+ [
82 "image", "inspect", "clang-format-service:latest"
84 with
open(os
.devnull
, "w") as f
:
85 subprocess
.check_call(command
, stdout
=f
)
87 print("I have to build the docker image first.")
88 self
.buildImage(docker_image
, cf_docker_folder
)
90 command
= self
.docker_command
+ [
91 'run', '-v', cwd
+ ':/home/worker/nss:Z', '--rm', '-ti', docker_image
93 # The clang format script returns 1 if something's to do. We don't
95 subprocess
.call(command
+ files
)
96 if self
.restorecon
is not None:
97 subprocess
.call([self
.restorecon
, '-R', cwd
])
99 def filesChanged(self
, path
):
101 for dirname
, dirnames
, files
in os
.walk(path
):
103 with
open(os
.path
.join(dirname
, file), "rb") as f
:
104 hash.update(f
.read())
105 chk_file
= cwd
+ "/.chk"
107 new_chk
= hash.hexdigest()
108 if os
.path
.exists(chk_file
):
109 with
open(chk_file
) as f
:
110 old_chk
= f
.readline()
111 if old_chk
!= new_chk
:
112 with
open(chk_file
, "w+") as f
:
117 def buildImage(self
, docker_image
, cf_docker_folder
):
118 command
= self
.docker_command
+ [
119 "build", "-t", docker_image
, cf_docker_folder
121 subprocess
.check_call(command
)
124 def setDockerCommand(self
, args
):
125 from distutils
.spawn
import find_executable
126 if platform
.system() == "Linux":
127 self
.restorecon
= find_executable("restorecon")
128 dcmd
= find_executable("docker")
130 self
.docker_command
= [dcmd
]
132 self
.docker_command
= ["sudo"] + self
.docker_command
134 self
.docker_command
= None
136 def modifiedFiles(self
):
138 if os
.path
.exists(os
.path
.join(cwd
, '.hg')):
139 st
= subprocess
.Popen(['hg', 'status', '-m', '-a'],
140 cwd
=cwd
, stdout
=subprocess
.PIPE
, universal_newlines
=True)
141 for line
in iter(st
.stdout
.readline
, ''):
142 files
+= [line
[2:].rstrip()]
143 elif os
.path
.exists(os
.path
.join(cwd
, '.git')):
144 st
= subprocess
.Popen(['git', 'status', '--porcelain'],
145 cwd
=cwd
, stdout
=subprocess
.PIPE
)
146 for line
in iter(st
.stdout
.readline
, ''):
147 if line
[1] == 'M' or line
[1] != 'D' and \
148 (line
[0] == 'M' or line
[0] == 'A' or
149 line
[0] == 'C' or line
[0] == 'U'):
150 files
+= [line
[3:].rstrip()]
152 files
+= [line
[line
.index(' -> ', beg
=4) + 4:]]
154 print('Warning: neither mercurial nor git detected!')
157 return x
[-2:] == '.c' or x
[-3:] == '.cc' or x
[-2:] == '.h'
158 return [x
for x
in files
if isFormatted(x
)]
161 class buildAction(argparse
.Action
):
163 def __call__(self
, parser
, args
, values
, option_string
=None):
164 subprocess
.check_call([cwd
+ "/build.sh"] + values
)
167 class testAction(argparse
.Action
):
169 def __call__(self
, parser
, args
, values
, option_string
=None):
173 class covAction(argparse
.Action
):
175 def runSslGtests(self
, outdir
):
177 "GTESTFILTER": "*", # Prevent parallel test runs.
178 "ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir
,
179 "NSS_DEFAULT_DB_TYPE": "sql",
180 "NSS_DISABLE_UNLOAD": "1"
183 run_tests("ssl_gtests", env
=env
, silent
=True)
185 def findSanCovFile(self
, outdir
):
186 for file in os
.listdir(outdir
):
187 if fnmatch
.fnmatch(file, 'ssl_gtest.*.sancov'):
188 return os
.path
.join(outdir
, file)
192 def __call__(self
, parser
, args
, values
, option_string
=None):
194 print("Output directory: " + outdir
)
196 print("\nBuild with coverage sanitizers...\n")
197 sancov_args
= "edge,no-prune,trace-pc-guard,trace-cmp"
198 subprocess
.check_call([
199 os
.path
.join(cwd
, "build.sh"), "-c", "--clang", "--asan", "--enable-legacy-db",
200 "--sancov=" + sancov_args
203 print("\nRun ssl_gtests to get a coverage report...")
204 self
.runSslGtests(outdir
)
207 sancov_file
= self
.findSanCovFile(outdir
)
209 print("Couldn't find .sancov file.")
212 symcov_file
= os
.path
.join(outdir
, "ssl_gtest.symcov")
213 out
= open(symcov_file
, 'wb')
214 # Don't exit immediately on error
215 symbol_retcode
= subprocess
.call([
217 "-blacklist=" + os
.path
.join(cwd
, ".sancov-blacklist"),
218 "-symbolize", sancov_file
,
219 os
.path
.join(cwd
, "../dist/Debug/bin/ssl_gtest")
223 print("\nCopying ssl_gtests to artifacts...")
224 shutil
.copyfile(os
.path
.join(cwd
, "../dist/Debug/bin/ssl_gtest"),
225 os
.path
.join(outdir
, "ssl_gtest"))
227 print("\nCoverage report: " + symcov_file
)
228 if symbol_retcode
> 0:
229 print("sancov failed to symbolize with return code {}".format(symbol_retcode
))
230 sys
.exit(symbol_retcode
)
232 class commandsAction(argparse
.Action
):
235 def __call__(self
, parser
, args
, values
, option_string
=None):
236 for c
in commandsAction
.commands
:
239 def parse_arguments():
240 parser
= argparse
.ArgumentParser(
241 description
='NSS helper script. ' +
242 'Make sure to separate sub-command arguments with --.')
243 subparsers
= parser
.add_subparsers()
245 parser_build
= subparsers
.add_parser(
246 'build', help='All arguments are passed to build.sh')
247 parser_build
.add_argument(
248 'build_args', nargs
='*', help="build arguments", action
=buildAction
)
250 parser_cf
= subparsers
.add_parser(
255 By default this runs against any files that you have modified. If
256 there are no modified files, it checks everything.
258 parser_cf
.add_argument(
260 help='On linux, suppress the use of \'sudo\' for running docker.',
262 parser_cf
.add_argument(
265 help="Specify files or directories to run clang-format on",
268 parser_test
= subparsers
.add_parser(
269 'tests', help='Run tests through tests/all.sh.')
271 "cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips",
272 "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec",
273 "gtests", "ssl_gtests", "bogo", "interop", "policy"
275 parser_test
.add_argument(
276 'test', choices
=tests
, help="Available tests", action
=testAction
)
278 parser_cov
= subparsers
.add_parser(
279 'coverage', help='Generate coverage report')
280 cov_modules
= ["ssl_gtests"]
281 parser_cov
.add_argument(
282 '--outdir', help='Output directory for coverage report data.',
283 default
=tempfile
.mkdtemp())
284 parser_cov
.add_argument(
285 'module', choices
=cov_modules
, help="Available coverage modules",
288 parser_commands
= subparsers
.add_parser(
290 help="list commands")
291 parser_commands
.add_argument(
294 action
=commandsAction
)
296 commandsAction
.commands
= [c
for c
in subparsers
.choices
]
297 return parser
.parse_args()
304 if __name__
== '__main__':