3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
21 from argparse
import Namespace
22 from collections
import defaultdict
, deque
, namedtuple
23 from contextlib
import contextmanager
24 from datetime
import datetime
, timedelta
25 from functools
import partial
26 from multiprocessing
import cpu_count
27 from subprocess
import PIPE
, STDOUT
, Popen
28 from tempfile
import gettempdir
, mkdtemp
29 from threading
import Event
, Thread
, Timer
, current_thread
33 from mozserve
import Http3Server
42 from xpcshellcommandline
import parser_desktop
44 SCRIPT_DIR
= os
.path
.abspath(os
.path
.realpath(os
.path
.dirname(__file__
)))
47 from mozbuild
.base
import MozbuildObject
49 build
= MozbuildObject
.from_environment(cwd
=SCRIPT_DIR
)
53 HARNESS_TIMEOUT
= 5 * 60
54 TBPL_RETRY
= 4 # defined in mozharness
56 # benchmarking on tbpl revealed that this works best for now
57 # TODO: This has been evaluated/set many years ago and we might want to
58 # benchmark this again.
59 # These days with e10s/fission the number of real processes/threads running
60 # can be significantly higher, with both consequences on runtime and memory
61 # consumption. So be aware that NUM_THREADS is just saying how many tests will
62 # be started maximum in parallel and that depending on the tests there is
63 # only a weak correlation to the effective number of processes or threads.
64 # Be also aware that we can override this value with the threadCount option
65 # on the command line to tweak it for a concrete CPU/memory combination.
66 NUM_THREADS
= int(cpu_count() * 4)
67 if sys
.platform
== "win32":
68 NUM_THREADS
= NUM_THREADS
/ 2
70 EXPECTED_LOG_ACTIONS
= set(
72 "crash_reporter_init",
78 # --------------------------------------------------------------
79 # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
81 here
= os
.path
.dirname(__file__
)
82 mozbase
= os
.path
.realpath(os
.path
.join(os
.path
.dirname(here
), "mozbase"))
84 if os
.path
.isdir(mozbase
):
85 for package
in os
.listdir(mozbase
):
86 sys
.path
.append(os
.path
.join(mozbase
, package
))
91 from manifestparser
import TestManifest
92 from manifestparser
.filters
import chunk_by_slice
, failures
, pathprefix
, tags
93 from manifestparser
.util
import normsep
94 from mozlog
import commandline
95 from mozprofile
import Profile
96 from mozprofile
.cli
import parse_key_value
, parse_preferences
97 from mozrunner
.utils
import get_stack_fixer_function
99 # --------------------------------------------------------------
101 # TODO: perhaps this should be in a more generally shared location?
102 # This regex matches all of the C0 and C1 control characters
103 # (U+0000 through U+001F; U+007F; U+0080 through U+009F),
104 # except TAB (U+0009), CR (U+000D), LF (U+000A) and backslash (U+005C).
105 # A raw string is deliberately not used.
106 _cleanup_encoding_re
= re
.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\\\\]")
109 def get_full_group_name(test
):
110 group
= test
["manifest"]
111 if "ancestor_manifest" in test
:
112 ancestor_manifest
= normsep(test
["ancestor_manifest"])
113 # Only change the group id if ancestor is not the generated root manifest.
114 if "/" in ancestor_manifest
:
115 group
= "{}:{}".format(ancestor_manifest
, group
)
119 def _cleanup_encoding_repl(m
):
121 return "\\\\" if c
== "\\" else "\\x{0:02X}".format(ord(c
))
124 def cleanup_encoding(s
):
125 """S is either a byte or unicode string. Either way it may
126 contain control characters, unpaired surrogates, reserved code
127 points, etc. If it is a byte string, it is assumed to be
128 UTF-8, but it may not be *correct* UTF-8. Return a
129 sanitized unicode object."""
130 if not isinstance(s
, six
.string_types
):
131 if isinstance(s
, six
.binary_type
):
132 return six
.ensure_str(s
)
134 return six
.text_type(s
)
135 if isinstance(s
, six
.binary_type
):
136 s
= s
.decode("utf-8", "replace")
137 # Replace all C0 and C1 control characters with \xNN escapes.
138 return _cleanup_encoding_re
.sub(_cleanup_encoding_repl
, s
)
142 def popenCleanupHack():
144 Hack to work around https://bugs.python.org/issue37380
145 The basic idea is that on old versions of Python on Windows,
146 we need to clear subprocess._cleanup before we call Popen(),
147 then restore it afterwards.
150 if mozinfo
.isWin
and sys
.version_info
[0] == 3 and sys
.version_info
< (3, 7, 5):
151 savedCleanup
= subprocess
._cleanup
152 subprocess
._cleanup
= lambda: None
157 subprocess
._cleanup
= savedCleanup
160 """ Control-C handling """
164 def markGotSIGINT(signum
, stackFrame
):
169 class XPCShellTestThread(Thread
):
176 usingCrashReporter
=False,
179 Thread
.__init
__(self
)
182 self
.test_object
= test_object
184 self
.verbose
= verbose
185 self
.usingTSan
= usingTSan
186 self
.usingCrashReporter
= usingCrashReporter
188 self
.appPath
= kwargs
.get("appPath")
189 self
.xrePath
= kwargs
.get("xrePath")
190 self
.utility_path
= kwargs
.get("utility_path")
191 self
.testingModulesDir
= kwargs
.get("testingModulesDir")
192 self
.debuggerInfo
= kwargs
.get("debuggerInfo")
193 self
.jsDebuggerInfo
= kwargs
.get("jsDebuggerInfo")
194 self
.headJSPath
= kwargs
.get("headJSPath")
195 self
.testharnessdir
= kwargs
.get("testharnessdir")
196 self
.profileName
= kwargs
.get("profileName")
197 self
.singleFile
= kwargs
.get("singleFile")
198 self
.env
= copy
.deepcopy(kwargs
.get("env"))
199 self
.symbolsPath
= kwargs
.get("symbolsPath")
200 self
.logfiles
= kwargs
.get("logfiles")
201 self
.app_binary
= kwargs
.get("app_binary")
202 self
.xpcshell
= kwargs
.get("xpcshell")
203 self
.xpcsRunArgs
= kwargs
.get("xpcsRunArgs")
204 self
.failureManifest
= kwargs
.get("failureManifest")
205 self
.jscovdir
= kwargs
.get("jscovdir")
206 self
.stack_fixer_function
= kwargs
.get("stack_fixer_function")
207 self
._rootTempDir
= kwargs
.get("tempDir")
208 self
.cleanup_dir_list
= kwargs
.get("cleanup_dir_list")
209 self
.pStdout
= kwargs
.get("pStdout")
210 self
.pStderr
= kwargs
.get("pStderr")
211 self
.keep_going
= kwargs
.get("keep_going")
212 self
.log
= kwargs
.get("log")
213 self
.app_dir_key
= kwargs
.get("app_dir_key")
214 self
.interactive
= kwargs
.get("interactive")
215 self
.rootPrefsFile
= kwargs
.get("rootPrefsFile")
216 self
.extraPrefs
= kwargs
.get("extraPrefs")
217 self
.verboseIfFails
= kwargs
.get("verboseIfFails")
218 self
.headless
= kwargs
.get("headless")
219 self
.runFailures
= kwargs
.get("runFailures")
220 self
.timeoutAsPass
= kwargs
.get("timeoutAsPass")
221 self
.crashAsPass
= kwargs
.get("crashAsPass")
222 self
.conditionedProfileDir
= kwargs
.get("conditionedProfileDir")
226 # Default the test prefsFile to the rootPrefsFile.
227 self
.prefsFile
= self
.rootPrefsFile
229 # only one of these will be set to 1. adding them to the totals in
235 # Context for output processing
236 self
.output_lines
= []
237 self
.has_failure_output
= False
238 self
.saw_crash_reporter_init
= False
239 self
.saw_proc_start
= False
240 self
.saw_proc_end
= False
242 self
.harness_timeout
= kwargs
.get("harness_timeout")
243 self
.timedout
= False
246 # event from main thread to signal work done
247 self
.event
= kwargs
.get("event")
248 self
.done
= False # explicitly set flag so we don't rely on thread.isAlive
253 except PermissionError
as e
:
256 self
.traceback
= traceback
.format_exc()
257 except Exception as e
:
259 self
.traceback
= traceback
.format_exc()
261 self
.exception
= None
262 self
.traceback
= None
265 "%s failed or timed out, will retry." % self
.test_object
["id"]
270 def kill(self
, proc
):
272 Simple wrapper to kill a process.
273 On a remote system, this is overloaded to handle remote process communication.
277 def removeDir(self
, dirname
):
279 Simple wrapper to remove (recursively) a given directory.
280 On a remote system, we need to overload this to work on the remote filesystem.
282 mozfile
.remove(dirname
)
284 def poll(self
, proc
):
286 Simple wrapper to check if a process has terminated.
287 On a remote system, this is overloaded to handle remote process communication.
291 def createLogFile(self
, test_file
, stdout
):
293 For a given test file and stdout buffer, create a log file.
294 On a remote system we have to fix the test name since it can contain directories.
296 with
open(test_file
+ ".log", "w") as f
:
299 def getReturnCode(self
, proc
):
301 Simple wrapper to get the return code for a given process.
302 On a remote system we overload this to work with the remote process management.
304 if proc
is not None and hasattr(proc
, "returncode"):
305 return proc
.returncode
308 def communicate(self
, proc
):
310 Simple wrapper to communicate with a process.
311 On a remote system, this is overloaded to handle remote process communication.
313 # Processing of incremental output put here to
314 # sidestep issues on remote platforms, where what we know
315 # as proc is a file pulled off of a device.
318 line
= proc
.stdout
.readline()
321 self
.process_line(line
)
323 if self
.saw_proc_start
and not self
.saw_proc_end
:
324 self
.has_failure_output
= True
326 return proc
.communicate()
329 self
, cmd
, stdout
, stderr
, env
, cwd
, timeout
=None, test_name
=None
332 Simple wrapper to launch a process.
333 On a remote system, this is more complex and we need to overload this function.
335 # timeout is needed by remote xpcshell to extend the
336 # remote device timeout. It is not used in this function.
338 cwd
= six
.ensure_str(cwd
)
339 for i
in range(len(cmd
)):
340 cmd
[i
] = six
.ensure_str(cmd
[i
])
343 popen_func
= psutil
.Popen
347 with
popenCleanupHack():
348 proc
= popen_func(cmd
, stdout
=stdout
, stderr
=stderr
, env
=env
, cwd
=cwd
)
352 def checkForCrashes(self
, dump_directory
, symbols_path
, test_name
=None):
354 Simple wrapper to check for crashes.
355 On a remote system, this is more complex and we need to overload this function.
361 return mozcrash
.log_crashes(
362 self
.log
, dump_directory
, symbols_path
, test
=test_name
, quiet
=quiet
365 def logCommand(self
, name
, completeCmd
, testdir
):
366 self
.log
.info("%s | full command: %r" % (name
, completeCmd
))
367 self
.log
.info("%s | current directory: %r" % (name
, testdir
))
368 # Show only those environment variables that are changed from
369 # the ambient environment.
370 changedEnv
= set("%s=%s" % i
for i
in six
.iteritems(self
.env
)) - set(
371 "%s=%s" % i
for i
in six
.iteritems(os
.environ
)
373 self
.log
.info("%s | environment: %s" % (name
, list(changedEnv
)))
374 shell_command_tokens
= [
375 pipes
.quote(tok
) for tok
in list(changedEnv
) + completeCmd
378 "%s | as shell command: (cd %s; %s)"
379 % (name
, pipes
.quote(testdir
), " ".join(shell_command_tokens
))
382 def killTimeout(self
, proc
):
383 if proc
is not None and hasattr(proc
, "pid"):
384 mozcrash
.kill_and_get_minidump(
385 proc
.pid
, self
.tempDir
, utility_path
=self
.utility_path
388 self
.log
.info("not killing -- proc or pid unknown")
390 def postCheck(self
, proc
):
391 """Checks for a still-running test process, kills it and fails the test if found.
392 We can sometimes get here before the process has terminated, which would
393 cause removeDir() to fail - so check for the process and kill it if needed.
395 if proc
and self
.poll(proc
) is None:
399 except psutil
.NoSuchProcess
:
403 message
= "%s | Process still running after test!" % self
.test_object
["id"]
405 self
.log
.info(message
)
408 self
.log
.error(message
)
409 self
.log_full_output()
412 def testTimeout(self
, proc
):
413 if self
.test_object
["expected"] == "pass":
420 self
.test_object
["id"],
423 message
="Test timed out",
427 if self
.timeoutAsPass
:
432 self
.test_object
["id"],
435 message
="Test timed out",
437 self
.log_full_output()
441 self
.killTimeout(proc
)
442 self
.log
.info("xpcshell return code: %s" % self
.getReturnCode(proc
))
444 self
.clean_temp_dirs(self
.test_object
["path"])
446 def updateTestPrefsFile(self
):
447 # If the Manifest file has some additional prefs, merge the
448 # prefs set in the user.js file stored in the _rootTempdir
449 # with the prefs from the manifest and the prefs specified
450 # in the extraPrefs option.
451 if "prefs" in self
.test_object
:
452 # Merge the user preferences in a fake profile dir in a
453 # local temporary dir (self.tempDir is the remoteTmpDir
454 # for the RemoteXPCShellTestThread subclass and so we
455 # can't use that tempDir here).
456 localTempDir
= mkdtemp(prefix
="xpc-other-", dir=self
._rootTempDir
)
459 interpolation
= {"server": "dummyserver"}
460 profile
= Profile(profile
=localTempDir
, restore
=False)
461 # _rootTempDir contains a user.js file, generated by buildPrefsFile
462 profile
.merge(self
._rootTempDir
, interpolation
=interpolation
)
464 prefs
= self
.test_object
["prefs"].strip().split()
465 name
= self
.test_object
["id"]
468 "%s: Per-test extra prefs will be set:\n {}".format(
474 profile
.set_preferences(parse_preferences(prefs
), filename
=filename
)
475 # Make sure that the extra prefs form the command line are overriding
476 # any prefs inherited from the shared profile data or the manifest prefs.
477 profile
.set_preferences(
478 parse_preferences(self
.extraPrefs
), filename
=filename
480 return os
.path
.join(profile
.profile
, filename
)
482 # Return the root prefsFile if there is no other prefs to merge.
483 # This is the path set by buildPrefsFile.
484 return self
.rootPrefsFile
487 def conditioned_profile_copy(self
):
488 """Returns a copy of the original conditioned profile that was created."""
490 condprof_copy
= os
.path
.join(tempfile
.mkdtemp(), "profile")
492 self
.conditionedProfileDir
,
494 ignore
=shutil
.ignore_patterns("lock"),
496 self
.log
.info("Created a conditioned-profile copy: %s" % condprof_copy
)
499 def buildCmdTestFile(self
, name
):
501 Build the command line arguments for the test file.
502 On a remote system, this may be overloaded to use a remote path structure.
504 return ["-e", 'const _TEST_FILE = ["%s"];' % name
.replace("\\", "/")]
506 def setupTempDir(self
):
507 tempDir
= mkdtemp(prefix
="xpc-other-", dir=self
._rootTempDir
)
508 self
.env
["XPCSHELL_TEST_TEMP_DIR"] = tempDir
510 self
.log
.info("temp dir is %s" % tempDir
)
513 def setupProfileDir(self
):
515 Create a temporary folder for the profile and set appropriate environment variables.
516 When running check-interactive and check-one, the directory is well-defined and
517 retained for inspection once the tests complete.
519 On a remote system, this may be overloaded to use a remote path structure.
521 if self
.conditionedProfileDir
:
522 profileDir
= self
.conditioned_profile_copy
523 elif self
.interactive
or self
.singleFile
:
524 profileDir
= os
.path
.join(gettempdir(), self
.profileName
, "xpcshellprofile")
526 # This could be left over from previous runs
527 self
.removeDir(profileDir
)
530 os
.makedirs(profileDir
)
532 profileDir
= mkdtemp(prefix
="xpc-profile-", dir=self
._rootTempDir
)
533 self
.env
["XPCSHELL_TEST_PROFILE_DIR"] = profileDir
534 if self
.interactive
or self
.singleFile
:
535 self
.log
.info("profile dir is %s" % profileDir
)
538 def setupMozinfoJS(self
):
539 mozInfoJSPath
= os
.path
.join(self
.profileDir
, "mozinfo.json")
540 mozInfoJSPath
= mozInfoJSPath
.replace("\\", "\\\\")
541 mozinfo
.output_to_file(mozInfoJSPath
)
544 def buildCmdHead(self
):
546 Build the command line arguments for the head files,
547 along with the address of the webserver which some tests require.
549 On a remote system, this is overloaded to resolve quoting issues over a
550 secondary command line.
552 headfiles
= self
.getHeadFiles(self
.test_object
)
553 cmdH
= ", ".join(['"' + f
.replace("\\", "/") + '"' for f
in headfiles
])
555 dbgport
= 0 if self
.jsDebuggerInfo
is None else self
.jsDebuggerInfo
.port
559 "const _HEAD_FILES = [%s];" % cmdH
,
561 "const _JSDEBUGGER_PORT = %d;" % dbgport
,
564 def getHeadFiles(self
, test
):
565 """Obtain lists of head- files. Returns a list of head files."""
567 def sanitize_list(s
, kind
):
568 for f
in s
.strip().split(" "):
573 path
= os
.path
.normpath(os
.path
.join(test
["here"], f
))
574 if not os
.path
.exists(path
):
575 raise Exception("%s file does not exist: %s" % (kind
, path
))
577 if not os
.path
.isfile(path
):
578 raise Exception("%s file is not a file: %s" % (kind
, path
))
582 headlist
= test
.get("head", "")
583 return list(sanitize_list(headlist
, "head"))
585 def buildXpcsCmd(self
):
587 Load the root head.js file as the first file in our test path, before other head,
588 and test files. On a remote system, we overload this to add additional command
589 line arguments, so this gets overloaded.
591 # - NOTE: if you rename/add any of the constants set here, update
592 # do_load_child_test_harness() in head.js
594 self
.appPath
= self
.xrePath
613 'const _HEAD_JS_PATH = "%s";' % self
.headJSPath
,
615 'const _MOZINFO_JS_PATH = "%s";' % self
.mozInfoJSPath
,
617 'const _PREFS_FILE = "%s";' % self
.prefsFile
.replace("\\", "\\\\"),
620 if self
.testingModulesDir
:
621 # Escape backslashes in string literal.
622 sanitized
= self
.testingModulesDir
.replace("\\", "\\\\")
623 xpcsCmd
.extend(["-e", 'const _TESTING_MODULES_DIR = "%s";' % sanitized
])
625 xpcsCmd
.extend(["-f", os
.path
.join(self
.testharnessdir
, "head.js")])
627 if self
.debuggerInfo
:
628 xpcsCmd
= [self
.debuggerInfo
.path
] + self
.debuggerInfo
.args
+ xpcsCmd
632 def cleanupDir(self
, directory
, name
):
633 if not os
.path
.exists(directory
):
636 # up to TRY_LIMIT attempts (one every second), because
637 # the Windows filesystem is slow to react to the changes
640 while try_count
< TRY_LIMIT
:
642 self
.removeDir(directory
)
644 self
.log
.info("Failed to remove directory: %s. Waiting." % directory
)
645 # We suspect the filesystem may still be making changes. Wait a
646 # little bit and try again.
653 # we try cleaning up again later at the end of the run
654 self
.cleanup_dir_list
.append(directory
)
656 def clean_temp_dirs(self
, name
):
657 # We don't want to delete the profile when running check-interactive
659 if self
.profileDir
and not self
.interactive
and not self
.singleFile
:
660 self
.cleanupDir(self
.profileDir
, name
)
662 self
.cleanupDir(self
.tempDir
, name
)
664 def parse_output(self
, output
):
665 """Parses process output for structured messages and saves output as it is
666 read. Sets self.has_failure_output in case of evidence of a failure"""
667 for line_string
in output
.splitlines():
668 self
.process_line(line_string
)
670 if self
.saw_proc_start
and not self
.saw_proc_end
:
671 self
.has_failure_output
= True
673 def fix_text_output(self
, line
):
674 line
= cleanup_encoding(line
)
675 if self
.stack_fixer_function
is not None:
676 line
= self
.stack_fixer_function(line
)
678 if isinstance(line
, bytes
):
679 line
= line
.decode("utf-8")
682 def log_line(self
, line
):
683 """Log a line of output (either a parser json object or text output from
685 if isinstance(line
, six
.string_types
) or isinstance(line
, bytes
):
686 line
= self
.fix_text_output(line
).rstrip("\r\n")
687 self
.log
.process_output(self
.proc_ident
, line
, command
=self
.command
)
689 if "message" in line
:
690 line
["message"] = self
.fix_text_output(line
["message"])
691 if "xpcshell_process" in line
:
692 line
["thread"] = " ".join(
693 [current_thread().name
, line
["xpcshell_process"]]
696 line
["thread"] = current_thread().name
697 self
.log
.log_raw(line
)
699 def log_full_output(self
):
700 """Logs any buffered output from the test process, and clears the buffer."""
701 if not self
.output_lines
:
703 self
.log
.info(">>>>>>>")
704 for line
in self
.output_lines
:
706 self
.log
.info("<<<<<<<")
707 self
.output_lines
= []
709 def report_message(self
, message
):
710 """Stores or logs a json log message in mozlog format."""
712 self
.log_line(message
)
714 self
.output_lines
.append(message
)
716 def process_line(self
, line_string
):
717 """Parses a single line of output, determining its significance and
720 if isinstance(line_string
, bytes
):
721 # Transform binary to string representation
722 line_string
= line_string
.decode(sys
.stdout
.encoding
, errors
="replace")
724 if not line_string
.strip():
728 line_object
= json
.loads(line_string
)
729 if not isinstance(line_object
, dict):
730 self
.report_message(line_string
)
733 self
.report_message(line_string
)
737 "action" not in line_object
738 or line_object
["action"] not in EXPECTED_LOG_ACTIONS
740 # The test process output JSON.
741 self
.report_message(line_string
)
744 if line_object
["action"] == "crash_reporter_init":
745 self
.saw_crash_reporter_init
= True
748 action
= line_object
["action"]
750 self
.has_failure_output
= (
751 self
.has_failure_output
752 or "expected" in line_object
754 and line_object
["level"] == "ERROR"
757 self
.report_message(line_object
)
759 if action
== "log" and line_object
["message"] == "CHILD-TEST-STARTED":
760 self
.saw_proc_start
= True
761 elif action
== "log" and line_object
["message"] == "CHILD-TEST-COMPLETED":
762 self
.saw_proc_end
= True
765 """Run an individual xpcshell test."""
768 name
= self
.test_object
["id"]
769 path
= self
.test_object
["path"]
770 group
= get_full_group_name(self
.test_object
)
772 # Check for skipped tests
773 if "disabled" in self
.test_object
:
774 message
= self
.test_object
["disabled"]
776 message
= "disabled from xpcshell manifest"
777 self
.log
.test_start(name
, group
=group
)
778 self
.log
.test_end(name
, "SKIP", message
=message
, group
=group
)
781 self
.keep_going
= True
784 # Check for known-fail tests
785 expect_pass
= self
.test_object
["expected"] == "pass"
787 # By default self.appPath will equal the gre dir. If specified in the
788 # xpcshell.toml file, set a different app dir for this test.
789 if self
.app_dir_key
and self
.app_dir_key
in self
.test_object
:
790 rel_app_dir
= self
.test_object
[self
.app_dir_key
]
791 rel_app_dir
= os
.path
.join(self
.xrePath
, rel_app_dir
)
792 self
.appPath
= os
.path
.abspath(rel_app_dir
)
796 test_dir
= os
.path
.dirname(path
)
798 # Create a profile and a temp dir that the JS harness can stick
799 # a profile and temporary data in
800 self
.profileDir
= self
.setupProfileDir()
801 self
.tempDir
= self
.setupTempDir()
802 self
.mozInfoJSPath
= self
.setupMozinfoJS()
804 # Setup per-manifest prefs and write them into the tempdir.
805 self
.prefsFile
= self
.updateTestPrefsFile()
807 # The order of the command line is important:
808 # 1) Arguments for xpcshell itself
809 self
.command
= self
.buildXpcsCmd()
811 # 2) Arguments for the head files
812 self
.command
.extend(self
.buildCmdHead())
814 # 3) Arguments for the test file
815 self
.command
.extend(self
.buildCmdTestFile(path
))
816 self
.command
.extend(["-e", 'const _TEST_NAME = "%s";' % name
])
818 # 4) Arguments for code coverage
821 ["-e", 'const _JSCOV_DIR = "%s";' % self
.jscovdir
.replace("\\", "/")]
824 # 5) Runtime arguments
825 if "debug" in self
.test_object
:
826 self
.command
.append("-d")
828 self
.command
.extend(self
.xpcsRunArgs
)
830 if self
.test_object
.get("dmd") == "true":
831 self
.env
["PYTHON"] = sys
.executable
832 self
.env
["BREAKPAD_SYMBOLS_PATH"] = self
.symbolsPath
834 if self
.test_object
.get("snap") == "true":
835 self
.env
["SNAP_NAME"] = "firefox"
836 self
.env
["SNAP_INSTANCE_NAME"] = "firefox"
838 if self
.test_object
.get("subprocess") == "true":
839 self
.env
["PYTHON"] = sys
.executable
842 self
.test_object
.get("headless", "true" if self
.headless
else None)
845 self
.env
["MOZ_HEADLESS"] = "1"
846 self
.env
["DISPLAY"] = "77" # Set a fake display.
848 testTimeoutInterval
= self
.harness_timeout
849 # Allow a test to request a multiple of the timeout if it is expected to take long
850 if "requesttimeoutfactor" in self
.test_object
:
851 testTimeoutInterval
*= int(self
.test_object
["requesttimeoutfactor"])
854 if not self
.interactive
and not self
.debuggerInfo
and not self
.jsDebuggerInfo
:
855 testTimer
= Timer(testTimeoutInterval
, lambda: self
.testTimeout(proc
))
859 process_output
= None
862 self
.log
.test_start(name
, group
=group
)
864 self
.logCommand(name
, self
.command
, test_dir
)
866 proc
= self
.launchProcess(
872 timeout
=testTimeoutInterval
,
876 if hasattr(proc
, "pid"):
877 self
.proc_ident
= proc
.pid
879 # On mobile, "proc" is just a file.
880 self
.proc_ident
= name
883 self
.log
.info("%s | Process ID: %d" % (name
, self
.proc_ident
))
885 # Communicate returns a tuple of (stdout, stderr), however we always
886 # redirect stderr to stdout, so the second element is ignored.
887 process_output
, _
= self
.communicate(proc
)
890 # Not sure what else to do here...
891 self
.keep_going
= True
898 # For the remote case, stdout is not yet depleted, so we parse
899 # it here all at once.
900 self
.parse_output(process_output
)
902 return_code
= self
.getReturnCode(proc
)
904 # TSan'd processes return 66 if races are detected. This isn't
905 # good in the sense that there's no way to distinguish between
906 # a process that would normally have returned zero but has races,
907 # and a race-free process that returns 66. But I don't see how
908 # to do better. This ambiguity is at least constrained to the
909 # with-TSan case. It doesn't affect normal builds.
911 # This also assumes that the magic value 66 isn't overridden by
912 # a TSAN_OPTIONS=exitcode=<number> environment variable setting.
914 TSAN_EXIT_CODE_WITH_RACES
= 66
916 return_code_ok
= return_code
== 0 or (
917 self
.usingTSan
and return_code
== TSAN_EXIT_CODE_WITH_RACES
920 # Due to the limitation on the remote xpcshell test, the process
921 # return code does not represent the process crash.
922 # If crash_reporter_init log has not been seen and the return code
923 # is 0, it means the process crashed before setting up the crash
926 # NOTE: Crash reporter is not enabled on some configuration, such
927 # as ASAN and TSAN. Those configuration shouldn't be using
928 # remote xpcshell test, and the crash should be caught by
929 # the process return code.
930 # NOTE: self.saw_crash_reporter_init is False also when adb failed
931 # to launch process, and in that case the return code is
933 # (see launchProcess in remotexpcshelltests.py)
934 ended_before_crash_reporter_init
= (
936 and self
.usingCrashReporter
937 and not self
.saw_crash_reporter_init
941 (not self
.has_failure_output
)
942 and not ended_before_crash_reporter_init
946 status
= "PASS" if passed
else "FAIL"
947 expected
= "PASS" if expect_pass
else "FAIL"
948 message
= "xpcshell return code: %d" % return_code
953 if status
!= expected
or ended_before_crash_reporter_init
:
954 if ended_before_crash_reporter_init
:
959 message
="Test ended before setting up the crash reporter",
967 message
="Test failed or timed out, will retry",
970 self
.clean_temp_dirs(path
)
971 if self
.verboseIfFails
and not self
.verbose
:
972 self
.log_full_output()
976 name
, status
, expected
=expected
, message
=message
, group
=group
978 self
.log_full_output()
982 if self
.failureManifest
:
983 with
open(self
.failureManifest
, "a") as f
:
984 f
.write("[%s]\n" % self
.test_object
["path"])
985 for k
, v
in self
.test_object
.items():
986 f
.write("%s = %s\n" % (k
, v
))
989 # If TSan reports a race, dump the output, else we can't
990 # diagnose what the problem was. See comments above about
991 # the significance of TSAN_EXIT_CODE_WITH_RACES.
992 if self
.usingTSan
and return_code
== TSAN_EXIT_CODE_WITH_RACES
:
993 self
.log_full_output()
996 name
, status
, expected
=expected
, message
=message
, group
=group
999 self
.log_full_output()
1008 if self
.checkForCrashes(self
.tempDir
, self
.symbolsPath
, test_name
=name
):
1010 self
.clean_temp_dirs(path
)
1013 # If we assert during shutdown there's a chance the test has passed
1014 # but we haven't logged full output, so do so here.
1015 self
.log_full_output()
1018 if self
.logfiles
and process_output
:
1019 self
.createLogFile(name
, process_output
)
1022 self
.postCheck(proc
)
1023 self
.clean_temp_dirs(path
)
1026 self
.log
.error("Received SIGINT (control-C) during test execution")
1030 self
.keep_going
= False
1033 self
.keep_going
= True
1036 class XPCShellTests(object):
1037 def __init__(self
, log
=None):
1038 """Initializes node status and logger."""
1040 self
.harness_timeout
= HARNESS_TIMEOUT
1042 self
.http3Server
= None
1043 self
.conditioned_profile_dir
= None
1045 def getTestManifest(self
, manifest
):
1046 if isinstance(manifest
, TestManifest
):
1048 elif manifest
is not None:
1049 manifest
= os
.path
.normpath(os
.path
.abspath(manifest
))
1050 if os
.path
.isfile(manifest
):
1051 return TestManifest([manifest
], strict
=True)
1053 toml_path
= os
.path
.join(manifest
, "xpcshell.toml")
1055 toml_path
= os
.path
.join(SCRIPT_DIR
, "tests", "xpcshell.toml")
1057 if os
.path
.exists(toml_path
):
1058 return TestManifest([toml_path
], strict
=True)
1061 "Failed to find manifest at %s; use --manifest "
1062 "to set path explicitly." % toml_path
1066 def normalizeTest(self
, root
, test_object
):
1067 path
= test_object
.get("file_relpath", test_object
["relpath"])
1068 if "dupe-manifest" in test_object
and "ancestor_manifest" in test_object
:
1069 test_object
["id"] = "%s:%s" % (
1070 os
.path
.basename(test_object
["ancestor_manifest"]),
1074 test_object
["id"] = path
1077 test_object
["manifest"] = os
.path
.relpath(test_object
["manifest"], root
)
1080 for key
in ("id", "manifest"):
1081 test_object
[key
] = test_object
[key
].replace(os
.sep
, "/")
1085 def buildTestList(self
, test_tags
=None, test_paths
=None, verify
=False):
1086 """Reads the xpcshell.toml manifest and set self.alltests to an array.
1088 Given the parameters, this method compiles a list of tests to be run
1089 that matches the criteria set by parameters.
1091 If any chunking of tests are to occur, it is also done in this method.
1093 If no tests are added to the list of tests to be run, an error
1094 is logged. A sys.exit() signal is sent to the caller.
1097 test_tags (list, optional): list of strings.
1098 test_paths (list, optional): list of strings derived from the command
1099 line argument provided by user, specifying
1101 verify (bool, optional): boolean value.
1103 if test_paths
is None:
1106 mp
= self
.getTestManifest(self
.manifest
)
1109 if build
and not root
:
1110 root
= build
.topsrcdir
1111 normalize
= partial(self
.normalizeTest
, root
)
1115 filters
.append(tags(test_tags
))
1119 path_filter
= pathprefix(test_paths
)
1120 filters
.append(path_filter
)
1122 noDefaultFilters
= False
1123 if self
.runFailures
:
1124 filters
.append(failures(self
.runFailures
))
1125 noDefaultFilters
= True
1127 if self
.totalChunks
> 1:
1128 filters
.append(chunk_by_slice(self
.thisChunk
, self
.totalChunks
))
1130 self
.alltests
= list(
1135 noDefaultFilters
=noDefaultFilters
,
1141 sys
.stderr
.write("*** offending mozinfo.info: %s\n" % repr(mozinfo
.info
))
1144 if path_filter
and path_filter
.missing
:
1146 "The following path(s) didn't resolve any tests:\n {}".format(
1147 " \n".join(sorted(path_filter
.missing
))
1151 if len(self
.alltests
) == 0:
1154 and path_filter
.missing
== set(test_paths
)
1155 and os
.environ
.get("MOZ_AUTOMATION") == "1"
1157 # This can happen in CI when a manifest doesn't exist due to a
1158 # build config variable in moz.build traversal. Don't generate
1159 # an error in this case. Adding a todo count avoids mozharness
1161 self
.todoCount
+= len(path_filter
.missing
)
1164 "no tests to run using specified "
1165 "combination of filters: {}".format(mp
.fmt_filters())
1169 if len(self
.alltests
) == 1 and not verify
:
1170 self
.singleFile
= os
.path
.basename(self
.alltests
[0]["path"])
1172 self
.singleFile
= None
1175 self
.dump_tests
= os
.path
.expanduser(self
.dump_tests
)
1176 assert os
.path
.exists(os
.path
.dirname(self
.dump_tests
))
1177 with
open(self
.dump_tests
, "w") as dumpFile
:
1178 dumpFile
.write(json
.dumps({"active_tests": self
.alltests
}))
1180 self
.log
.info("Dumping active_tests to %s file." % self
.dump_tests
)
1183 def setAbsPath(self
):
1185 Set the absolute path for xpcshell and xrepath. These 3 variables
1186 depend on input from the command line and we need to allow for absolute paths.
1187 This function is overloaded for a remote solution as os.path* won't work remotely.
1189 self
.testharnessdir
= os
.path
.dirname(os
.path
.abspath(__file__
))
1190 self
.headJSPath
= self
.testharnessdir
.replace("\\", "/") + "/head.js"
1191 if self
.xpcshell
is not None:
1192 self
.xpcshell
= os
.path
.abspath(self
.xpcshell
)
1194 if self
.app_binary
is not None:
1195 self
.app_binary
= os
.path
.abspath(self
.app_binary
)
1197 if self
.xrePath
is None:
1198 binary_path
= self
.app_binary
or self
.xpcshell
1199 self
.xrePath
= os
.path
.dirname(binary_path
)
1201 # Check if we're run from an OSX app bundle and override
1202 # self.xrePath if we are.
1203 appBundlePath
= os
.path
.join(
1204 os
.path
.dirname(os
.path
.dirname(self
.xpcshell
)), "Resources"
1206 if os
.path
.exists(os
.path
.join(appBundlePath
, "application.ini")):
1207 self
.xrePath
= appBundlePath
1209 self
.xrePath
= os
.path
.abspath(self
.xrePath
)
1211 if self
.mozInfo
is None:
1212 self
.mozInfo
= os
.path
.join(self
.testharnessdir
, "mozinfo.json")
1214 def buildPrefsFile(self
, extraPrefs
):
1215 # Create the prefs.js file
1217 # In test packages used in CI, the profile_data directory is installed
1218 # in the SCRIPT_DIR.
1219 profile_data_dir
= os
.path
.join(SCRIPT_DIR
, "profile_data")
1220 # If possible, read profile data from topsrcdir. This prevents us from
1221 # requiring a re-build to pick up newly added extensions in the
1222 # <profile>/extensions directory.
1224 path
= os
.path
.join(build
.topsrcdir
, "testing", "profiles")
1225 if os
.path
.isdir(path
):
1226 profile_data_dir
= path
1227 # Still not found? Look for testing/profiles relative to testing/xpcshell.
1228 if not os
.path
.isdir(profile_data_dir
):
1229 path
= os
.path
.abspath(os
.path
.join(SCRIPT_DIR
, "..", "profiles"))
1230 if os
.path
.isdir(path
):
1231 profile_data_dir
= path
1233 with
open(os
.path
.join(profile_data_dir
, "profiles.json"), "r") as fh
:
1234 base_profiles
= json
.load(fh
)["xpcshell"]
1236 # values to use when interpolating preferences
1238 "server": "dummyserver",
1241 profile
= Profile(profile
=self
.tempDir
, restore
=False)
1242 prefsFile
= os
.path
.join(profile
.profile
, "user.js")
1244 # Empty the user.js file in case the file existed before.
1245 with
open(prefsFile
, "w"):
1248 for name
in base_profiles
:
1249 path
= os
.path
.join(profile_data_dir
, name
)
1250 profile
.merge(path
, interpolation
=interpolation
)
1252 # add command line prefs
1253 prefs
= parse_preferences(extraPrefs
)
1254 profile
.set_preferences(prefs
)
1256 self
.prefsFile
= prefsFile
1259 def buildCoreEnvironment(self
):
1261 Add environment variables likely to be used across all platforms, including
1264 # Make assertions fatal
1265 self
.env
["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
1266 # Crash reporting interferes with debugging
1267 if not self
.debuggerInfo
:
1268 self
.env
["MOZ_CRASHREPORTER"] = "1"
1269 # Don't launch the crash reporter client
1270 self
.env
["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
1271 # Don't permit remote connections by default.
1272 # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
1273 # enable non-local connections for the purposes of local testing.
1274 # Don't override the user's choice here. See bug 1049688.
1275 self
.env
.setdefault("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "1")
1276 if self
.mozInfo
.get("topsrcdir") is not None:
1277 self
.env
["MOZ_DEVELOPER_REPO_DIR"] = self
.mozInfo
["topsrcdir"]
1278 if self
.mozInfo
.get("topobjdir") is not None:
1279 self
.env
["MOZ_DEVELOPER_OBJ_DIR"] = self
.mozInfo
["topobjdir"]
1281 # Disable the content process sandbox for the xpcshell tests. They
1282 # currently attempt to do things like bind() sockets, which is not
1283 # compatible with the sandbox.
1284 self
.env
["MOZ_DISABLE_CONTENT_SANDBOX"] = "1"
1285 if os
.getenv("MOZ_FETCHES_DIR", None):
1286 self
.env
["MOZ_FETCHES_DIR"] = os
.getenv("MOZ_FETCHES_DIR", None)
1288 if self
.mozInfo
.get("socketprocess_networking"):
1289 self
.env
["MOZ_FORCE_USE_SOCKET_PROCESS"] = "1"
1291 self
.env
["MOZ_DISABLE_SOCKET_PROCESS"] = "1"
1293 def buildEnvironment(self
):
1295 Create and returns a dictionary of self.env to include all the appropriate env
1296 variables and values. On a remote system, we overload this to set different
1297 values and are missing things like os.environ and PATH.
1299 self
.env
= dict(os
.environ
)
1300 self
.buildCoreEnvironment()
1301 if sys
.platform
== "win32":
1302 self
.env
["PATH"] = self
.env
["PATH"] + ";" + self
.xrePath
1303 elif sys
.platform
in ("os2emx", "os2knix"):
1304 os
.environ
["BEGINLIBPATH"] = self
.xrePath
+ ";" + self
.env
["BEGINLIBPATH"]
1305 os
.environ
["LIBPATHSTRICT"] = "T"
1306 elif sys
.platform
== "osx" or sys
.platform
== "darwin":
1307 self
.env
["DYLD_LIBRARY_PATH"] = os
.path
.join(
1308 os
.path
.dirname(self
.xrePath
), "MacOS"
1310 else: # unix or linux?
1311 if "LD_LIBRARY_PATH" not in self
.env
or self
.env
["LD_LIBRARY_PATH"] is None:
1312 self
.env
["LD_LIBRARY_PATH"] = self
.xrePath
1314 self
.env
["LD_LIBRARY_PATH"] = ":".join(
1315 [self
.xrePath
, self
.env
["LD_LIBRARY_PATH"]]
1318 usingASan
= "asan" in self
.mozInfo
and self
.mozInfo
["asan"]
1319 usingTSan
= "tsan" in self
.mozInfo
and self
.mozInfo
["tsan"]
1320 if usingASan
or usingTSan
:
1321 # symbolizer support
1322 if "ASAN_SYMBOLIZER_PATH" in self
.env
and os
.path
.isfile(
1323 self
.env
["ASAN_SYMBOLIZER_PATH"]
1325 llvmsym
= self
.env
["ASAN_SYMBOLIZER_PATH"]
1327 llvmsym
= os
.path
.join(
1328 self
.xrePath
, "llvm-symbolizer" + self
.mozInfo
["bin_suffix"]
1330 if os
.path
.isfile(llvmsym
):
1332 self
.env
["ASAN_SYMBOLIZER_PATH"] = llvmsym
1334 oldTSanOptions
= self
.env
.get("TSAN_OPTIONS", "")
1335 self
.env
["TSAN_OPTIONS"] = "external_symbolizer_path={} {}".format(
1336 llvmsym
, oldTSanOptions
1338 self
.log
.info("runxpcshelltests.py | using symbolizer at %s" % llvmsym
)
1341 "TEST-UNEXPECTED-FAIL | runxpcshelltests.py | "
1342 "Failed to find symbolizer at %s" % llvmsym
1349 Determine the value of the stdout and stderr for the test.
1350 Return value is a list (pStdout, pStderr).
1352 if self
.interactive
:
1356 if self
.debuggerInfo
and self
.debuggerInfo
.interactive
:
1360 if sys
.platform
== "os2emx":
1365 return pStdout
, pStderr
1367 def verifyDirPath(self
, dirname
):
1369 Simple wrapper to get the absolute path for a given directory name.
1370 On a remote system, we need to overload this to work on the remote filesystem.
1372 return os
.path
.abspath(dirname
)
1374 def trySetupNode(self
):
1376 Run node for HTTP/2 tests, if available, and updates mozinfo as appropriate.
1378 if os
.getenv("MOZ_ASSUME_NODE_RUNNING", None):
1379 self
.log
.info("Assuming required node servers are already running")
1380 if not os
.getenv("MOZHTTP2_PORT", None):
1382 "MOZHTTP2_PORT environment variable not set. "
1383 "Tests requiring http/2 will fail."
1387 # We try to find the node executable in the path given to us by the user in
1388 # the MOZ_NODE_PATH environment variable
1389 nodeBin
= os
.getenv("MOZ_NODE_PATH", None)
1390 if not nodeBin
and build
:
1391 nodeBin
= build
.substs
.get("NODEJS")
1394 "MOZ_NODE_PATH environment variable not set. "
1395 "Tests requiring http/2 will fail."
1399 if not os
.path
.exists(nodeBin
) or not os
.path
.isfile(nodeBin
):
1400 error
= "node not found at MOZ_NODE_PATH %s" % (nodeBin
)
1401 self
.log
.error(error
)
1402 raise IOError(error
)
1404 self
.log
.info("Found node at %s" % (nodeBin
,))
1406 def read_streams(name
, proc
, pipe
):
1407 output
= "stdout" if pipe
== proc
.stdout
else "stderr"
1408 for line
in iter(pipe
.readline
, ""):
1409 self
.log
.info("node %s [%s] %s" % (name
, output
, line
))
1411 def startServer(name
, serverJs
):
1412 if not os
.path
.exists(serverJs
):
1413 error
= "%s not found at %s" % (name
, serverJs
)
1414 self
.log
.error(error
)
1415 raise IOError(error
)
1417 # OK, we found our server, let's try to get it running
1418 self
.log
.info("Found %s at %s" % (name
, serverJs
))
1420 # We pipe stdin to node because the server will exit when its
1422 with
popenCleanupHack():
1424 [nodeBin
, serverJs
],
1430 universal_newlines
=True,
1431 start_new_session
=True,
1433 self
.nodeProc
[name
] = process
1435 # Check to make sure the server starts properly by waiting for it to
1436 # tell us it's started
1437 msg
= process
.stdout
.readline()
1438 if "server listening" in msg
:
1439 searchObj
= re
.search(
1440 r
"HTTP2 server listening on ports ([0-9]+),([0-9]+)", msg
, 0
1443 self
.env
["MOZHTTP2_PORT"] = searchObj
.group(1)
1444 self
.env
["MOZNODE_EXEC_PORT"] = searchObj
.group(2)
1446 target
=read_streams
,
1447 args
=(name
, process
, process
.stdout
),
1452 target
=read_streams
,
1453 args
=(name
, process
, process
.stderr
),
1457 except OSError as e
:
1458 # This occurs if the subprocess couldn't be started
1459 self
.log
.error("Could not run %s server: %s" % (name
, str(e
)))
1462 myDir
= os
.path
.split(os
.path
.abspath(__file__
))[0]
1463 startServer("moz-http2", os
.path
.join(myDir
, "moz-http2", "moz-http2.js"))
1465 def shutdownNode(self
):
1467 Shut down our node process, if it exists
1469 for name
, proc
in six
.iteritems(self
.nodeProc
):
1470 self
.log
.info("Node %s server shutting down ..." % name
)
1471 if proc
.poll() is not None:
1472 self
.log
.info("Node server %s already dead %s" % (name
, proc
.poll()))
1473 elif sys
.platform
!= "win32":
1474 # Kill process and all its spawned children.
1475 os
.killpg(proc
.pid
, signal
.SIGTERM
)
1481 def startHttp3Server(self
):
1483 Start a Http3 test server.
1486 if sys
.platform
== "win32":
1488 http3ServerPath
= self
.http3ServerPath
1489 if not http3ServerPath
:
1490 http3ServerPath
= os
.path
.join(
1491 SCRIPT_DIR
, "http3server", "http3server" + binSuffix
1494 http3ServerPath
= os
.path
.join(
1495 build
.topobjdir
, "dist", "bin", "http3server" + binSuffix
1497 dbPath
= os
.path
.join(SCRIPT_DIR
, "http3server", "http3serverDB")
1499 dbPath
= os
.path
.join(build
.topsrcdir
, "netwerk", "test", "http3serverDB")
1501 options
["http3ServerPath"] = http3ServerPath
1502 options
["profilePath"] = dbPath
1503 options
["isMochitest"] = False
1504 options
["isWin"] = sys
.platform
== "win32"
1505 serverEnv
= self
.env
.copy()
1506 serverLog
= self
.env
.get("MOZHTTP3_SERVER_LOG")
1507 if serverLog
is not None:
1508 serverEnv
["RUST_LOG"] = serverLog
1509 self
.http3Server
= Http3Server(options
, serverEnv
, self
.log
)
1510 self
.http3Server
.start()
1511 for key
, value
in self
.http3Server
.ports().items():
1512 self
.env
[key
] = value
1513 self
.env
["MOZHTTP3_ECH"] = self
.http3Server
.echConfig()
1515 def shutdownHttp3Server(self
):
1516 if self
.http3Server
is None:
1518 self
.http3Server
.stop()
1519 self
.http3Server
= None
1521 def buildXpcsRunArgs(self
):
1523 Add arguments to run the test or make it interactive.
1525 if self
.interactive
:
1526 self
.xpcsRunArgs
= [
1528 'print("To start the test, type |_execute_test();|.");',
1532 self
.xpcsRunArgs
= ["-e", "_execute_test(); quit(0);"]
1534 def addTestResults(self
, test
):
1535 self
.passCount
+= test
.passCount
1536 self
.failCount
+= test
.failCount
1537 self
.todoCount
+= test
.todoCount
1539 def updateMozinfo(self
, prefs
, options
):
1540 # Handle filenames in mozInfo
1541 if not isinstance(self
.mozInfo
, dict):
1542 mozInfoFile
= self
.mozInfo
1543 if not os
.path
.isfile(mozInfoFile
):
1545 "Error: couldn't find mozinfo.json at '%s'. Perhaps you "
1546 "need to use --build-info-json?" % mozInfoFile
1549 self
.mozInfo
= json
.load(open(mozInfoFile
))
1551 # mozinfo.info is used as kwargs. Some builds are done with
1552 # an older Python that can't handle Unicode keys in kwargs.
1553 # All of the keys in question should be ASCII.
1555 for k
, v
in self
.mozInfo
.items():
1556 if isinstance(k
, bytes
):
1557 k
= k
.decode("utf-8")
1559 self
.mozInfo
= fixedInfo
1561 self
.mozInfo
["fission"] = prefs
.get("fission.autostart", True)
1562 self
.mozInfo
["sessionHistoryInParent"] = self
.mozInfo
[
1564 ] or not prefs
.get("fission.disableSessionHistoryInParent", False)
1566 self
.mozInfo
["serviceworker_e10s"] = True
1568 self
.mozInfo
["verify"] = options
.get("verify", False)
1570 self
.mozInfo
["socketprocess_networking"] = prefs
.get(
1571 "network.http.network_access_on_socket_process.enabled", False
1574 self
.mozInfo
["condprof"] = options
.get("conditionedProfile", False)
1576 if options
.get("variant", ""):
1577 self
.mozInfo
["msix"] = options
["variant"] == "msix"
1579 self
.mozInfo
["is_ubuntu"] = "Ubuntu" in platform
.version()
1581 mozinfo
.update(self
.mozInfo
)
1586 def conditioned_profile_copy(self
):
1587 """Returns a copy of the original conditioned profile that was created."""
1588 condprof_copy
= os
.path
.join(tempfile
.mkdtemp(), "profile")
1590 self
.conditioned_profile_dir
,
1592 ignore
=shutil
.ignore_patterns("lock"),
1594 self
.log
.info("Created a conditioned-profile copy: %s" % condprof_copy
)
1595 return condprof_copy
1597 def downloadConditionedProfile(self
, profile_scenario
, app
):
1598 from condprof
.client
import get_profile
1599 from condprof
.util
import get_current_platform
, get_version
1601 if self
.conditioned_profile_dir
:
1602 # We already have a directory, so provide a copy that
1603 # will get deleted after it's done with
1604 return self
.conditioned_profile_dir
1606 # create a temp file to help ensure uniqueness
1607 temp_download_dir
= tempfile
.mkdtemp()
1609 "Making temp_download_dir from inside get_conditioned_profile {}".format(
1613 # call condprof's client API to yield our platform-specific
1614 # conditioned-profile binary
1615 platform
= get_current_platform()
1617 if isinstance(app
, str):
1618 version
= get_version(app
)
1620 if not profile_scenario
:
1621 profile_scenario
= "settled"
1623 cond_prof_target_dir
= get_profile(
1627 repo
="mozilla-central",
1633 # any other error is a showstopper
1634 self
.log
.critical("Could not get the conditioned profile")
1635 traceback
.print_exc()
1639 self
.log
.info("Retrying a profile with no version specified")
1640 cond_prof_target_dir
= get_profile(
1644 repo
="mozilla-central",
1648 self
.log
.critical("Could not get the conditioned profile")
1649 traceback
.print_exc()
1652 # now get the full directory path to our fetched conditioned profile
1653 self
.conditioned_profile_dir
= os
.path
.join(
1654 temp_download_dir
, cond_prof_target_dir
1656 if not os
.path
.exists(cond_prof_target_dir
):
1658 "Can't find target_dir {}, from get_profile()"
1659 "temp_download_dir {}, platform {}, scenario {}".format(
1660 cond_prof_target_dir
, temp_download_dir
, platform
, profile_scenario
1666 "Original self.conditioned_profile_dir is now set: {}".format(
1667 self
.conditioned_profile_dir
1670 return self
.conditioned_profile_copy
1672 def runSelfTest(self
):
1679 class XPCShellTestsTests(selftest
.XPCShellTestsTests
):
1680 def __init__(self
, name
):
1681 unittest
.TestCase
.__init
__(self
, name
)
1682 self
.testing_modules
= this
.testingModulesDir
1683 self
.xpcshellBin
= this
.xpcshell
1684 self
.app_binary
= this
.app_binary
1685 self
.utility_path
= this
.utility_path
1686 self
.symbols_path
= this
.symbolsPath
1688 old_info
= dict(mozinfo
.info
)
1690 suite
= unittest
.TestLoader().loadTestsFromTestCase(XPCShellTestsTests
)
1691 return unittest
.TextTestRunner(verbosity
=2).run(suite
).wasSuccessful()
1693 # The self tests modify mozinfo, so we need to reset it.
1694 mozinfo
.info
.clear()
1695 mozinfo
.update(old_info
)
1697 def runTests(self
, options
, testClass
=XPCShellTestThread
, mobileArgs
=None):
1703 # Number of times to repeat test(s) in --verify mode
1706 if isinstance(options
, Namespace
):
1707 options
= vars(options
)
1709 # Try to guess modules directory.
1710 # This somewhat grotesque hack allows the buildbot machines to find the
1711 # modules directory without having to configure the buildbot hosts. This
1712 # code path should never be executed in local runs because the build system
1713 # should always set this argument.
1714 if not options
.get("testingModulesDir"):
1715 possible
= os
.path
.join(here
, os
.path
.pardir
, "modules")
1717 if os
.path
.isdir(possible
):
1718 testingModulesDir
= possible
1720 if options
.get("rerun_failures"):
1721 if os
.path
.exists(options
.get("failure_manifest")):
1722 rerun_manifest
= os
.path
.join(
1723 os
.path
.dirname(options
["failure_manifest"]), "rerun.toml"
1725 shutil
.copyfile(options
["failure_manifest"], rerun_manifest
)
1726 os
.remove(options
["failure_manifest"])
1728 self
.log
.error("No failures were found to re-run.")
1731 if options
.get("testingModulesDir"):
1732 # The resource loader expects native paths. Depending on how we were
1733 # invoked, a UNIX style path may sneak in on Windows. We try to
1735 testingModulesDir
= os
.path
.normpath(options
["testingModulesDir"])
1737 if not os
.path
.isabs(testingModulesDir
):
1738 testingModulesDir
= os
.path
.abspath(testingModulesDir
)
1740 if not testingModulesDir
.endswith(os
.path
.sep
):
1741 testingModulesDir
+= os
.path
.sep
1743 self
.debuggerInfo
= None
1745 if options
.get("debugger"):
1746 self
.debuggerInfo
= mozdebug
.get_debugger_info(
1747 options
.get("debugger"),
1748 options
.get("debuggerArgs"),
1749 options
.get("debuggerInteractive"),
1752 self
.jsDebuggerInfo
= None
1753 if options
.get("jsDebugger"):
1754 # A namedtuple let's us keep .port instead of ['port']
1755 JSDebuggerInfo
= namedtuple("JSDebuggerInfo", ["port"])
1756 self
.jsDebuggerInfo
= JSDebuggerInfo(port
=options
["jsDebuggerPort"])
1758 self
.app_binary
= options
.get("app_binary")
1759 self
.xpcshell
= options
.get("xpcshell")
1760 self
.http3ServerPath
= options
.get("http3server")
1761 self
.xrePath
= options
.get("xrePath")
1762 self
.utility_path
= options
.get("utility_path")
1763 self
.appPath
= options
.get("appPath")
1764 self
.symbolsPath
= options
.get("symbolsPath")
1765 self
.tempDir
= os
.path
.normpath(options
.get("tempDir") or tempfile
.gettempdir())
1766 self
.manifest
= options
.get("manifest")
1767 self
.dump_tests
= options
.get("dump_tests")
1768 self
.interactive
= options
.get("interactive")
1769 self
.verbose
= options
.get("verbose")
1770 self
.verboseIfFails
= options
.get("verboseIfFails")
1771 self
.keepGoing
= options
.get("keepGoing")
1772 self
.logfiles
= options
.get("logfiles")
1773 self
.totalChunks
= options
.get("totalChunks", 1)
1774 self
.thisChunk
= options
.get("thisChunk")
1775 self
.profileName
= options
.get("profileName") or "xpcshell"
1776 self
.mozInfo
= options
.get("mozInfo")
1777 self
.testingModulesDir
= testingModulesDir
1778 self
.sequential
= options
.get("sequential")
1779 self
.failure_manifest
= options
.get("failure_manifest")
1780 self
.threadCount
= options
.get("threadCount") or NUM_THREADS
1781 self
.jscovdir
= options
.get("jscovdir")
1782 self
.headless
= options
.get("headless")
1783 self
.runFailures
= options
.get("runFailures")
1784 self
.timeoutAsPass
= options
.get("timeoutAsPass")
1785 self
.crashAsPass
= options
.get("crashAsPass")
1786 self
.conditionedProfile
= options
.get("conditionedProfile")
1787 self
.repeat
= options
.get("repeat", 0)
1788 self
.variant
= options
.get("variant", "")
1790 if self
.variant
== "msix":
1791 self
.appPath
= options
.get("msixAppPath")
1792 self
.xrePath
= options
.get("msixXrePath")
1793 self
.app_binary
= options
.get("msix_app_binary")
1794 self
.threadCount
= 2
1795 self
.xpcshell
= None
1802 if self
.conditionedProfile
:
1803 self
.conditioned_profile_dir
= self
.downloadConditionedProfile(
1804 "full", self
.appPath
1806 options
["self_test"] = False
1807 if not options
["test_tags"]:
1808 options
["test_tags"] = []
1809 options
["test_tags"].append("condprof")
1813 eprefs
= options
.get("extraPrefs") or []
1814 # enable fission by default
1815 if options
.get("disableFission"):
1816 eprefs
.append("fission.autostart=false")
1818 # should be by default, just in case
1819 eprefs
.append("fission.autostart=true")
1821 prefs
= self
.buildPrefsFile(eprefs
)
1822 self
.buildXpcsRunArgs()
1824 self
.event
= Event()
1826 if not self
.updateMozinfo(prefs
, options
):
1830 "These variables are available in the mozinfo environment and "
1831 "can be used to skip tests conditionally:"
1833 for info
in sorted(self
.mozInfo
.items(), key
=lambda item
: item
[0]):
1834 self
.log
.info(" {key}: {value}".format(key
=info
[0], value
=info
[1]))
1836 if options
.get("self_test"):
1837 if not self
.runSelfTest():
1841 ("tsan" in self
.mozInfo
and self
.mozInfo
["tsan"])
1842 or ("asan" in self
.mozInfo
and self
.mozInfo
["asan"])
1843 ) and not options
.get("threadCount"):
1844 # TSan/ASan require significantly more memory, so reduce the amount of parallel
1845 # tests we run to avoid OOMs and timeouts. We always keep a minimum of 2 for
1846 # non-sequential execution.
1847 # pylint --py3k W1619
1848 self
.threadCount
= max(self
.threadCount
/ 2, 2)
1850 self
.stack_fixer_function
= None
1851 if self
.utility_path
and os
.path
.exists(self
.utility_path
):
1852 self
.stack_fixer_function
= get_stack_fixer_function(
1853 self
.utility_path
, self
.symbolsPath
1856 # buildEnvironment() needs mozInfo, so we call it after mozInfo is initialized.
1857 self
.buildEnvironment()
1858 extraEnv
= parse_key_value(options
.get("extraEnv") or [], context
="--setenv")
1859 for k
, v
in extraEnv
:
1862 "Using environment variable %s instead of %s." % (v
, self
.env
[k
])
1866 # The appDirKey is a optional entry in either the default or individual test
1867 # sections that defines a relative application directory for test runs. If
1868 # defined we pass 'grePath/$appDirKey' for the -a parameter of the xpcshell
1871 if "appname" in self
.mozInfo
:
1872 appDirKey
= self
.mozInfo
["appname"] + "-appdir"
1874 # We have to do this before we run tests that depend on having the node
1878 self
.startHttp3Server()
1880 pStdout
, pStderr
= self
.getPipes()
1883 options
.get("test_tags"), options
.get("testPaths"), options
.get("verify")
1886 self
.sequential
= True
1888 if options
.get("shuffle"):
1889 random
.shuffle(self
.alltests
)
1891 self
.cleanup_dir_list
= []
1894 "appPath": self
.appPath
,
1895 "xrePath": self
.xrePath
,
1896 "utility_path": self
.utility_path
,
1897 "testingModulesDir": self
.testingModulesDir
,
1898 "debuggerInfo": self
.debuggerInfo
,
1899 "jsDebuggerInfo": self
.jsDebuggerInfo
,
1900 "headJSPath": self
.headJSPath
,
1901 "tempDir": self
.tempDir
,
1902 "testharnessdir": self
.testharnessdir
,
1903 "profileName": self
.profileName
,
1904 "singleFile": self
.singleFile
,
1905 "env": self
.env
, # making a copy of this in the testthreads
1906 "symbolsPath": self
.symbolsPath
,
1907 "logfiles": self
.logfiles
,
1908 "app_binary": self
.app_binary
,
1909 "xpcshell": self
.xpcshell
,
1910 "xpcsRunArgs": self
.xpcsRunArgs
,
1911 "failureManifest": self
.failure_manifest
,
1912 "jscovdir": self
.jscovdir
,
1913 "harness_timeout": self
.harness_timeout
,
1914 "stack_fixer_function": self
.stack_fixer_function
,
1915 "event": self
.event
,
1916 "cleanup_dir_list": self
.cleanup_dir_list
,
1919 "keep_going": self
.keepGoing
,
1921 "interactive": self
.interactive
,
1922 "app_dir_key": appDirKey
,
1923 "rootPrefsFile": self
.prefsFile
,
1924 "extraPrefs": options
.get("extraPrefs") or [],
1925 "verboseIfFails": self
.verboseIfFails
,
1926 "headless": self
.headless
,
1927 "runFailures": self
.runFailures
,
1928 "timeoutAsPass": self
.timeoutAsPass
,
1929 "crashAsPass": self
.crashAsPass
,
1930 "conditionedProfileDir": self
.conditioned_profile_dir
,
1931 "repeat": self
.repeat
,
1935 # Allow user to kill hung xpcshell subprocess with SIGINT
1936 # when we are only running tests sequentially.
1937 signal
.signal(signal
.SIGINT
, markGotSIGINT
)
1939 if self
.debuggerInfo
:
1940 # Force a sequential run
1941 self
.sequential
= True
1943 # If we have an interactive debugger, disable SIGINT entirely.
1944 if self
.debuggerInfo
.interactive
:
1945 signal
.signal(signal
.SIGINT
, lambda signum
, frame
: None)
1947 if "lldb" in self
.debuggerInfo
.path
:
1948 # Ask people to start debugging using 'process launch', see bug 952211.
1950 "It appears that you're using LLDB to debug this test. "
1951 + "Please use the 'process launch' command instead of "
1952 "the 'run' command to start xpcshell."
1955 if self
.jsDebuggerInfo
:
1956 # The js debugger magic needs more work to do the right thing
1957 # if debugging multiple files.
1958 if len(self
.alltests
) != 1:
1960 "Error: --jsdebugger can only be used with a single test!"
1964 # The test itself needs to know whether it is a tsan build, since
1965 # that has an effect on interpretation of the process return value.
1966 usingTSan
= "tsan" in self
.mozInfo
and self
.mozInfo
["tsan"]
1968 usingCrashReporter
= (
1969 "crashreporter" in self
.mozInfo
and self
.mozInfo
["crashreporter"]
1972 # create a queue of all tests that will run
1973 tests_queue
= deque()
1974 # also a list for the tests that need to be run sequentially
1975 sequential_tests
= []
1978 if options
.get("repeat", 0) > 0:
1979 self
.sequential
= True
1981 if not options
.get("verify"):
1982 for test_object
in self
.alltests
:
1983 # Test identifiers are provided for the convenience of logging. These
1984 # start as path names but are rewritten in case tests from the same path
1987 path
= test_object
["path"]
1989 if self
.singleFile
and not path
.endswith(self
.singleFile
):
1992 # if we have --repeat, duplicate the tests as needed
1993 for i
in range(0, options
.get("repeat", 0) + 1):
1998 verbose
=self
.verbose
or test_object
.get("verbose") == "true",
1999 usingTSan
=usingTSan
,
2000 usingCrashReporter
=usingCrashReporter
,
2001 mobileArgs
=mobileArgs
,
2004 if "run-sequentially" in test_object
or self
.sequential
:
2005 sequential_tests
.append(test
)
2007 tests_queue
.append(test
)
2009 status
= self
.runTestList(
2010 tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2014 # Test verification: Run each test many times, in various configurations,
2015 # in hopes of finding intermittent failures.
2019 # Run tests sequentially. Parallel mode would also work, except that
2020 # the logging system gets confused when 2 or more tests with the same
2021 # name run at the same time.
2022 sequential_tests
= []
2023 for i
in range(VERIFY_REPEAT
):
2026 test_object
, retry
=False, mobileArgs
=mobileArgs
, **kwargs
2028 sequential_tests
.append(test
)
2029 status
= self
.runTestList(
2030 tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2035 # Run tests sequentially, with MOZ_CHAOSMODE enabled.
2036 sequential_tests
= []
2037 self
.env
["MOZ_CHAOSMODE"] = "0xfb"
2038 # chaosmode runs really slow, allow tests extra time to pass
2039 kwargs
["harness_timeout"] = self
.harness_timeout
* 2
2040 for i
in range(VERIFY_REPEAT
):
2043 test_object
, retry
=False, mobileArgs
=mobileArgs
, **kwargs
2045 sequential_tests
.append(test
)
2046 status
= self
.runTestList(
2047 tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2049 kwargs
["harness_timeout"] = self
.harness_timeout
2053 ("1. Run each test %d times, sequentially." % VERIFY_REPEAT
, step1
),
2055 "2. Run each test %d times, sequentially, in chaos mode."
2060 startTime
= datetime
.now()
2061 maxTime
= timedelta(seconds
=options
["verifyMaxTime"])
2062 for test_object
in self
.alltests
:
2064 for descr
, step
in steps
:
2065 stepResults
[descr
] = "not run / incomplete"
2066 finalResult
= "PASSED"
2067 for descr
, step
in steps
:
2068 if (datetime
.now() - startTime
) > maxTime
:
2070 "::: Test verification is taking too long: Giving up!"
2073 "::: So far, all checks passed, but not "
2074 "all checks were run."
2077 self
.log
.info(":::")
2078 self
.log
.info('::: Running test verification step "%s"...' % descr
)
2079 self
.log
.info(":::")
2081 if status
is not True:
2082 stepResults
[descr
] = "FAIL"
2083 finalResult
= "FAILED!"
2085 stepResults
[descr
] = "Pass"
2086 self
.log
.info(":::")
2088 "::: Test verification summary for: %s" % test_object
["path"]
2090 self
.log
.info(":::")
2091 for descr
in sorted(stepResults
.keys()):
2092 self
.log
.info("::: %s : %s" % (descr
, stepResults
[descr
]))
2093 self
.log
.info(":::")
2094 self
.log
.info("::: Test verification %s" % finalResult
)
2095 self
.log
.info(":::")
2098 self
.shutdownHttp3Server()
2102 def start_test(self
, test
):
2105 def test_ended(self
, test
):
2109 self
, tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2112 self
.log
.info("Running tests sequentially.")
2114 self
.log
.info("Using at most %d threads." % self
.threadCount
)
2116 # keep a set of threadCount running tests and start running the
2117 # tests in the queue at most threadCount at a time
2118 running_tests
= set()
2123 self
.try_again_list
= []
2125 tests_by_manifest
= defaultdict(list)
2126 for test
in self
.alltests
:
2127 group
= get_full_group_name(test
)
2128 tests_by_manifest
[group
].append(test
["id"])
2130 self
.log
.suite_start(tests_by_manifest
, name
="xpcshell")
2132 while tests_queue
or running_tests
:
2133 # if we're not supposed to continue and all of the running tests
2135 if not keep_going
and not running_tests
:
2138 # if there's room to run more tests, start running them
2140 keep_going
and tests_queue
and (len(running_tests
) < self
.threadCount
)
2142 test
= tests_queue
.popleft()
2143 running_tests
.add(test
)
2144 self
.start_test(test
)
2146 # queue is full (for now) or no more new tests,
2147 # process the finished tests so far
2149 # wait for at least one of the tests to finish
2153 # find what tests are done (might be more than 1)
2155 for test
in running_tests
:
2157 self
.test_ended(test
)
2158 done_tests
.add(test
)
2161 ) # join with timeout so we don't hang on blocked threads
2162 # if the test had trouble, we will try running it again
2163 # at the end of the run
2164 if test
.retry
or test
.is_alive():
2165 # if the join call timed out, test.is_alive => True
2166 self
.try_again_list
.append(test
.test_object
)
2168 # did the test encounter any exception?
2170 exceptions
.append(test
.exception
)
2171 tracebacks
.append(test
.traceback
)
2172 # we won't add any more tests, will just wait for
2173 # the currently running ones to finish
2175 infra_abort
= infra_abort
and test
.infra
2176 keep_going
= keep_going
and test
.keep_going
2177 self
.addTestResults(test
)
2179 # make room for new tests to run
2180 running_tests
.difference_update(done_tests
)
2183 return TBPL_RETRY
# terminate early
2186 # run the other tests sequentially
2187 for test
in sequential_tests
:
2190 "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so "
2191 "stopped run. (Use --keep-going to keep running tests "
2192 "after killing one with SIGINT)"
2195 self
.start_test(test
)
2197 self
.test_ended(test
)
2198 if (test
.failCount
> 0 or test
.passCount
<= 0) and os
.environ
.get(
2201 self
.try_again_list
.append(test
.test_object
)
2203 self
.addTestResults(test
)
2204 # did the test encounter any exception?
2206 exceptions
.append(test
.exception
)
2207 tracebacks
.append(test
.traceback
)
2209 keep_going
= test
.keep_going
2211 # retry tests that failed when run in parallel
2212 if self
.try_again_list
:
2213 self
.log
.info("Retrying tests that failed when run in parallel.")
2214 for test_object
in self
.try_again_list
:
2218 verbose
=self
.verbose
,
2219 mobileArgs
=mobileArgs
,
2222 self
.start_test(test
)
2224 self
.test_ended(test
)
2225 self
.addTestResults(test
)
2226 # did the test encounter any exception?
2228 exceptions
.append(test
.exception
)
2229 tracebacks
.append(test
.traceback
)
2231 keep_going
= test
.keep_going
2233 # restore default SIGINT behaviour
2234 signal
.signal(signal
.SIGINT
, signal
.SIG_DFL
)
2236 # Clean up any slacker directories that might be lying around
2237 # Some might fail because of windows taking too long to unlock them.
2238 # We don't do anything if this fails because the test machines will have
2239 # their $TEMP dirs cleaned up on reboot anyway.
2240 for directory
in self
.cleanup_dir_list
:
2242 shutil
.rmtree(directory
)
2244 self
.log
.info("%s could not be cleaned up." % directory
)
2247 self
.log
.info("Following exceptions were raised:")
2248 for t
in tracebacks
:
2252 if self
.testCount
== 0 and os
.environ
.get("MOZ_AUTOMATION") != "1":
2253 self
.log
.error("No tests run. Did you pass an invalid --test-path?")
2256 # doing this allows us to pass the mozharness parsers that
2257 # report an orange job for failCount>0
2258 if self
.runFailures
:
2259 passed
= self
.passCount
2260 self
.passCount
= self
.failCount
2261 self
.failCount
= passed
2263 self
.log
.info("INFO | Result summary:")
2264 self
.log
.info("INFO | Passed: %d" % self
.passCount
)
2265 self
.log
.info("INFO | Failed: %d" % self
.failCount
)
2266 self
.log
.info("INFO | Todo: %d" % self
.todoCount
)
2267 self
.log
.info("INFO | Retried: %d" % len(self
.try_again_list
))
2269 if gotSIGINT
and not keep_going
:
2271 "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. "
2272 "(Use --keep-going to keep running tests after "
2273 "killing one with SIGINT)"
2277 self
.log
.suite_end()
2278 return self
.runFailures
or self
.failCount
== 0
2282 parser
= parser_desktop()
2283 options
= parser
.parse_args()
2285 log
= commandline
.setup_logging("XPCShell", options
, {"tbpl": sys
.stdout
})
2287 if options
.xpcshell
is None and options
.app_binary
is None:
2289 "Must provide path to xpcshell using --xpcshell or Firefox using --app-binary"
2293 if options
.xpcshell
is not None and options
.app_binary
is not None:
2295 "Cannot provide --xpcshell and --app-binary - they are mutually exclusive options. Choose one."
2299 xpcsh
= XPCShellTests(log
)
2301 if options
.interactive
and not options
.testPath
:
2302 log
.error("Error: You must specify a test filename in interactive mode!")
2305 result
= xpcsh
.runTests(options
)
2306 if result
== TBPL_RETRY
:
2313 if __name__
== "__main__":