3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
21 from argparse
import Namespace
22 from collections
import defaultdict
, deque
, namedtuple
23 from contextlib
import contextmanager
24 from datetime
import datetime
, timedelta
25 from functools
import partial
26 from multiprocessing
import cpu_count
27 from subprocess
import PIPE
, STDOUT
, Popen
28 from tempfile
import gettempdir
, mkdtemp
29 from threading
import Event
, Thread
, Timer
, current_thread
33 from mozserve
import Http3Server
42 from xpcshellcommandline
import parser_desktop
44 SCRIPT_DIR
= os
.path
.abspath(os
.path
.realpath(os
.path
.dirname(__file__
)))
47 from mozbuild
.base
import MozbuildObject
49 build
= MozbuildObject
.from_environment(cwd
=SCRIPT_DIR
)
53 HARNESS_TIMEOUT
= 5 * 60
54 TBPL_RETRY
= 4 # defined in mozharness
56 # benchmarking on tbpl revealed that this works best for now
57 # TODO: This has been evaluated/set many years ago and we might want to
58 # benchmark this again.
59 # These days with e10s/fission the number of real processes/threads running
60 # can be significantly higher, with both consequences on runtime and memory
61 # consumption. So be aware that NUM_THREADS is just saying how many tests will
62 # be started maximum in parallel and that depending on the tests there is
63 # only a weak correlation to the effective number of processes or threads.
64 # Be also aware that we can override this value with the threadCount option
65 # on the command line to tweak it for a concrete CPU/memory combination.
66 NUM_THREADS
= int(cpu_count() * 4)
67 if sys
.platform
== "win32":
68 NUM_THREADS
= NUM_THREADS
/ 2
70 EXPECTED_LOG_ACTIONS
= set(
72 "crash_reporter_init",
78 # --------------------------------------------------------------
79 # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
81 here
= os
.path
.dirname(__file__
)
82 mozbase
= os
.path
.realpath(os
.path
.join(os
.path
.dirname(here
), "mozbase"))
84 if os
.path
.isdir(mozbase
):
85 for package
in os
.listdir(mozbase
):
86 sys
.path
.append(os
.path
.join(mozbase
, package
))
91 from manifestparser
import TestManifest
92 from manifestparser
.filters
import chunk_by_slice
, failures
, pathprefix
, tags
93 from manifestparser
.util
import normsep
94 from mozlog
import commandline
95 from mozprofile
import Profile
96 from mozprofile
.cli
import parse_preferences
97 from mozrunner
.utils
import get_stack_fixer_function
99 # --------------------------------------------------------------
101 # TODO: perhaps this should be in a more generally shared location?
102 # This regex matches all of the C0 and C1 control characters
103 # (U+0000 through U+001F; U+007F; U+0080 through U+009F),
104 # except TAB (U+0009), CR (U+000D), LF (U+000A) and backslash (U+005C).
105 # A raw string is deliberately not used.
106 _cleanup_encoding_re
= re
.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\\\\]")
109 def get_full_group_name(test
):
110 group
= test
["manifest"]
111 if "ancestor_manifest" in test
:
112 ancestor_manifest
= normsep(test
["ancestor_manifest"])
113 # Only change the group id if ancestor is not the generated root manifest.
114 if "/" in ancestor_manifest
:
115 group
= "{}:{}".format(ancestor_manifest
, group
)
119 def _cleanup_encoding_repl(m
):
121 return "\\\\" if c
== "\\" else "\\x{0:02X}".format(ord(c
))
124 def cleanup_encoding(s
):
125 """S is either a byte or unicode string. Either way it may
126 contain control characters, unpaired surrogates, reserved code
127 points, etc. If it is a byte string, it is assumed to be
128 UTF-8, but it may not be *correct* UTF-8. Return a
129 sanitized unicode object."""
130 if not isinstance(s
, six
.string_types
):
131 if isinstance(s
, six
.binary_type
):
132 return six
.ensure_str(s
)
134 return six
.text_type(s
)
135 if isinstance(s
, six
.binary_type
):
136 s
= s
.decode("utf-8", "replace")
137 # Replace all C0 and C1 control characters with \xNN escapes.
138 return _cleanup_encoding_re
.sub(_cleanup_encoding_repl
, s
)
142 def popenCleanupHack():
144 Hack to work around https://bugs.python.org/issue37380
145 The basic idea is that on old versions of Python on Windows,
146 we need to clear subprocess._cleanup before we call Popen(),
147 then restore it afterwards.
150 if mozinfo
.isWin
and sys
.version_info
[0] == 3 and sys
.version_info
< (3, 7, 5):
151 savedCleanup
= subprocess
._cleanup
152 subprocess
._cleanup
= lambda: None
157 subprocess
._cleanup
= savedCleanup
160 """ Control-C handling """
164 def markGotSIGINT(signum
, stackFrame
):
169 class XPCShellTestThread(Thread
):
176 usingCrashReporter
=False,
179 Thread
.__init
__(self
)
182 self
.test_object
= test_object
184 self
.verbose
= verbose
185 self
.usingTSan
= usingTSan
186 self
.usingCrashReporter
= usingCrashReporter
188 self
.appPath
= kwargs
.get("appPath")
189 self
.xrePath
= kwargs
.get("xrePath")
190 self
.utility_path
= kwargs
.get("utility_path")
191 self
.testingModulesDir
= kwargs
.get("testingModulesDir")
192 self
.debuggerInfo
= kwargs
.get("debuggerInfo")
193 self
.jsDebuggerInfo
= kwargs
.get("jsDebuggerInfo")
194 self
.headJSPath
= kwargs
.get("headJSPath")
195 self
.testharnessdir
= kwargs
.get("testharnessdir")
196 self
.profileName
= kwargs
.get("profileName")
197 self
.singleFile
= kwargs
.get("singleFile")
198 self
.env
= copy
.deepcopy(kwargs
.get("env"))
199 self
.symbolsPath
= kwargs
.get("symbolsPath")
200 self
.logfiles
= kwargs
.get("logfiles")
201 self
.app_binary
= kwargs
.get("app_binary")
202 self
.xpcshell
= kwargs
.get("xpcshell")
203 self
.xpcsRunArgs
= kwargs
.get("xpcsRunArgs")
204 self
.failureManifest
= kwargs
.get("failureManifest")
205 self
.jscovdir
= kwargs
.get("jscovdir")
206 self
.stack_fixer_function
= kwargs
.get("stack_fixer_function")
207 self
._rootTempDir
= kwargs
.get("tempDir")
208 self
.cleanup_dir_list
= kwargs
.get("cleanup_dir_list")
209 self
.pStdout
= kwargs
.get("pStdout")
210 self
.pStderr
= kwargs
.get("pStderr")
211 self
.keep_going
= kwargs
.get("keep_going")
212 self
.log
= kwargs
.get("log")
213 self
.app_dir_key
= kwargs
.get("app_dir_key")
214 self
.interactive
= kwargs
.get("interactive")
215 self
.rootPrefsFile
= kwargs
.get("rootPrefsFile")
216 self
.extraPrefs
= kwargs
.get("extraPrefs")
217 self
.verboseIfFails
= kwargs
.get("verboseIfFails")
218 self
.headless
= kwargs
.get("headless")
219 self
.runFailures
= kwargs
.get("runFailures")
220 self
.timeoutAsPass
= kwargs
.get("timeoutAsPass")
221 self
.crashAsPass
= kwargs
.get("crashAsPass")
222 self
.conditionedProfileDir
= kwargs
.get("conditionedProfileDir")
226 # Default the test prefsFile to the rootPrefsFile.
227 self
.prefsFile
= self
.rootPrefsFile
229 # only one of these will be set to 1. adding them to the totals in
235 # Context for output processing
236 self
.output_lines
= []
237 self
.has_failure_output
= False
238 self
.saw_crash_reporter_init
= False
239 self
.saw_proc_start
= False
240 self
.saw_proc_end
= False
242 self
.harness_timeout
= kwargs
.get("harness_timeout")
243 self
.timedout
= False
246 # event from main thread to signal work done
247 self
.event
= kwargs
.get("event")
248 self
.done
= False # explicitly set flag so we don't rely on thread.isAlive
253 except PermissionError
as e
:
256 self
.traceback
= traceback
.format_exc()
257 except Exception as e
:
259 self
.traceback
= traceback
.format_exc()
261 self
.exception
= None
262 self
.traceback
= None
265 "%s failed or timed out, will retry." % self
.test_object
["id"]
270 def kill(self
, proc
):
272 Simple wrapper to kill a process.
273 On a remote system, this is overloaded to handle remote process communication.
277 def removeDir(self
, dirname
):
279 Simple wrapper to remove (recursively) a given directory.
280 On a remote system, we need to overload this to work on the remote filesystem.
282 mozfile
.remove(dirname
)
284 def poll(self
, proc
):
286 Simple wrapper to check if a process has terminated.
287 On a remote system, this is overloaded to handle remote process communication.
291 def createLogFile(self
, test_file
, stdout
):
293 For a given test file and stdout buffer, create a log file.
294 On a remote system we have to fix the test name since it can contain directories.
296 with
open(test_file
+ ".log", "w") as f
:
299 def getReturnCode(self
, proc
):
301 Simple wrapper to get the return code for a given process.
302 On a remote system we overload this to work with the remote process management.
304 if proc
is not None and hasattr(proc
, "returncode"):
305 return proc
.returncode
308 def communicate(self
, proc
):
310 Simple wrapper to communicate with a process.
311 On a remote system, this is overloaded to handle remote process communication.
313 # Processing of incremental output put here to
314 # sidestep issues on remote platforms, where what we know
315 # as proc is a file pulled off of a device.
318 line
= proc
.stdout
.readline()
321 self
.process_line(line
)
323 if self
.saw_proc_start
and not self
.saw_proc_end
:
324 self
.has_failure_output
= True
326 return proc
.communicate()
329 self
, cmd
, stdout
, stderr
, env
, cwd
, timeout
=None, test_name
=None
332 Simple wrapper to launch a process.
333 On a remote system, this is more complex and we need to overload this function.
335 # timeout is needed by remote xpcshell to extend the
336 # remote device timeout. It is not used in this function.
338 cwd
= six
.ensure_str(cwd
)
339 for i
in range(len(cmd
)):
340 cmd
[i
] = six
.ensure_str(cmd
[i
])
343 popen_func
= psutil
.Popen
347 with
popenCleanupHack():
348 proc
= popen_func(cmd
, stdout
=stdout
, stderr
=stderr
, env
=env
, cwd
=cwd
)
352 def checkForCrashes(self
, dump_directory
, symbols_path
, test_name
=None):
354 Simple wrapper to check for crashes.
355 On a remote system, this is more complex and we need to overload this function.
361 return mozcrash
.log_crashes(
362 self
.log
, dump_directory
, symbols_path
, test
=test_name
, quiet
=quiet
365 def logCommand(self
, name
, completeCmd
, testdir
):
366 self
.log
.info("%s | full command: %r" % (name
, completeCmd
))
367 self
.log
.info("%s | current directory: %r" % (name
, testdir
))
368 # Show only those environment variables that are changed from
369 # the ambient environment.
370 changedEnv
= set("%s=%s" % i
for i
in six
.iteritems(self
.env
)) - set(
371 "%s=%s" % i
for i
in six
.iteritems(os
.environ
)
373 self
.log
.info("%s | environment: %s" % (name
, list(changedEnv
)))
374 shell_command_tokens
= [
375 pipes
.quote(tok
) for tok
in list(changedEnv
) + completeCmd
378 "%s | as shell command: (cd %s; %s)"
379 % (name
, pipes
.quote(testdir
), " ".join(shell_command_tokens
))
382 def killTimeout(self
, proc
):
383 if proc
is not None and hasattr(proc
, "pid"):
384 mozcrash
.kill_and_get_minidump(
385 proc
.pid
, self
.tempDir
, utility_path
=self
.utility_path
388 self
.log
.info("not killing -- proc or pid unknown")
390 def postCheck(self
, proc
):
391 """Checks for a still-running test process, kills it and fails the test if found.
392 We can sometimes get here before the process has terminated, which would
393 cause removeDir() to fail - so check for the process and kill it if needed.
395 if proc
and self
.poll(proc
) is None:
399 except psutil
.NoSuchProcess
:
403 message
= "%s | Process still running after test!" % self
.test_object
["id"]
405 self
.log
.info(message
)
408 self
.log
.error(message
)
409 self
.log_full_output()
412 def testTimeout(self
, proc
):
413 if self
.test_object
["expected"] == "pass":
420 self
.test_object
["id"],
423 message
="Test timed out",
427 if self
.timeoutAsPass
:
432 self
.test_object
["id"],
435 message
="Test timed out",
437 self
.log_full_output()
441 self
.killTimeout(proc
)
442 self
.log
.info("xpcshell return code: %s" % self
.getReturnCode(proc
))
444 self
.clean_temp_dirs(self
.test_object
["path"])
446 def updateTestPrefsFile(self
):
447 # If the Manifest file has some additional prefs, merge the
448 # prefs set in the user.js file stored in the _rootTempdir
449 # with the prefs from the manifest and the prefs specified
450 # in the extraPrefs option.
451 if "prefs" in self
.test_object
:
452 # Merge the user preferences in a fake profile dir in a
453 # local temporary dir (self.tempDir is the remoteTmpDir
454 # for the RemoteXPCShellTestThread subclass and so we
455 # can't use that tempDir here).
456 localTempDir
= mkdtemp(prefix
="xpc-other-", dir=self
._rootTempDir
)
459 interpolation
= {"server": "dummyserver"}
460 profile
= Profile(profile
=localTempDir
, restore
=False)
461 # _rootTempDir contains a user.js file, generated by buildPrefsFile
462 profile
.merge(self
._rootTempDir
, interpolation
=interpolation
)
464 prefs
= self
.test_object
["prefs"].strip().split()
465 name
= self
.test_object
["id"]
468 "%s: Per-test extra prefs will be set:\n {}".format(
474 profile
.set_preferences(parse_preferences(prefs
), filename
=filename
)
475 # Make sure that the extra prefs form the command line are overriding
476 # any prefs inherited from the shared profile data or the manifest prefs.
477 profile
.set_preferences(
478 parse_preferences(self
.extraPrefs
), filename
=filename
480 return os
.path
.join(profile
.profile
, filename
)
482 # Return the root prefsFile if there is no other prefs to merge.
483 # This is the path set by buildPrefsFile.
484 return self
.rootPrefsFile
487 def conditioned_profile_copy(self
):
488 """Returns a copy of the original conditioned profile that was created."""
490 condprof_copy
= os
.path
.join(tempfile
.mkdtemp(), "profile")
492 self
.conditionedProfileDir
,
494 ignore
=shutil
.ignore_patterns("lock"),
496 self
.log
.info("Created a conditioned-profile copy: %s" % condprof_copy
)
499 def buildCmdTestFile(self
, name
):
501 Build the command line arguments for the test file.
502 On a remote system, this may be overloaded to use a remote path structure.
504 return ["-e", 'const _TEST_FILE = ["%s"];' % name
.replace("\\", "/")]
506 def setupTempDir(self
):
507 tempDir
= mkdtemp(prefix
="xpc-other-", dir=self
._rootTempDir
)
508 self
.env
["XPCSHELL_TEST_TEMP_DIR"] = tempDir
510 self
.log
.info("temp dir is %s" % tempDir
)
513 def setupProfileDir(self
):
515 Create a temporary folder for the profile and set appropriate environment variables.
516 When running check-interactive and check-one, the directory is well-defined and
517 retained for inspection once the tests complete.
519 On a remote system, this may be overloaded to use a remote path structure.
521 if self
.conditionedProfileDir
:
522 profileDir
= self
.conditioned_profile_copy
523 elif self
.interactive
or self
.singleFile
:
524 profileDir
= os
.path
.join(gettempdir(), self
.profileName
, "xpcshellprofile")
526 # This could be left over from previous runs
527 self
.removeDir(profileDir
)
530 os
.makedirs(profileDir
)
532 profileDir
= mkdtemp(prefix
="xpc-profile-", dir=self
._rootTempDir
)
533 self
.env
["XPCSHELL_TEST_PROFILE_DIR"] = profileDir
534 if self
.interactive
or self
.singleFile
:
535 self
.log
.info("profile dir is %s" % profileDir
)
538 def setupMozinfoJS(self
):
539 mozInfoJSPath
= os
.path
.join(self
.profileDir
, "mozinfo.json")
540 mozInfoJSPath
= mozInfoJSPath
.replace("\\", "\\\\")
541 mozinfo
.output_to_file(mozInfoJSPath
)
544 def buildCmdHead(self
):
546 Build the command line arguments for the head files,
547 along with the address of the webserver which some tests require.
549 On a remote system, this is overloaded to resolve quoting issues over a
550 secondary command line.
552 headfiles
= self
.getHeadFiles(self
.test_object
)
553 cmdH
= ", ".join(['"' + f
.replace("\\", "/") + '"' for f
in headfiles
])
555 dbgport
= 0 if self
.jsDebuggerInfo
is None else self
.jsDebuggerInfo
.port
559 "const _HEAD_FILES = [%s];" % cmdH
,
561 "const _JSDEBUGGER_PORT = %d;" % dbgport
,
564 def getHeadFiles(self
, test
):
565 """Obtain lists of head- files. Returns a list of head files."""
567 def sanitize_list(s
, kind
):
568 for f
in s
.strip().split(" "):
573 path
= os
.path
.normpath(os
.path
.join(test
["here"], f
))
574 if not os
.path
.exists(path
):
575 raise Exception("%s file does not exist: %s" % (kind
, path
))
577 if not os
.path
.isfile(path
):
578 raise Exception("%s file is not a file: %s" % (kind
, path
))
582 headlist
= test
.get("head", "")
583 return list(sanitize_list(headlist
, "head"))
585 def buildXpcsCmd(self
):
587 Load the root head.js file as the first file in our test path, before other head,
588 and test files. On a remote system, we overload this to add additional command
589 line arguments, so this gets overloaded.
591 # - NOTE: if you rename/add any of the constants set here, update
592 # do_load_child_test_harness() in head.js
594 self
.appPath
= self
.xrePath
613 'const _HEAD_JS_PATH = "%s";' % self
.headJSPath
,
615 'const _MOZINFO_JS_PATH = "%s";' % self
.mozInfoJSPath
,
617 'const _PREFS_FILE = "%s";' % self
.prefsFile
.replace("\\", "\\\\"),
620 if self
.testingModulesDir
:
621 # Escape backslashes in string literal.
622 sanitized
= self
.testingModulesDir
.replace("\\", "\\\\")
623 xpcsCmd
.extend(["-e", 'const _TESTING_MODULES_DIR = "%s";' % sanitized
])
625 xpcsCmd
.extend(["-f", os
.path
.join(self
.testharnessdir
, "head.js")])
627 if self
.debuggerInfo
:
628 xpcsCmd
= [self
.debuggerInfo
.path
] + self
.debuggerInfo
.args
+ xpcsCmd
632 def cleanupDir(self
, directory
, name
):
633 if not os
.path
.exists(directory
):
636 # up to TRY_LIMIT attempts (one every second), because
637 # the Windows filesystem is slow to react to the changes
640 while try_count
< TRY_LIMIT
:
642 self
.removeDir(directory
)
644 self
.log
.info("Failed to remove directory: %s. Waiting." % directory
)
645 # We suspect the filesystem may still be making changes. Wait a
646 # little bit and try again.
653 # we try cleaning up again later at the end of the run
654 self
.cleanup_dir_list
.append(directory
)
656 def clean_temp_dirs(self
, name
):
657 # We don't want to delete the profile when running check-interactive
659 if self
.profileDir
and not self
.interactive
and not self
.singleFile
:
660 self
.cleanupDir(self
.profileDir
, name
)
662 self
.cleanupDir(self
.tempDir
, name
)
664 def parse_output(self
, output
):
665 """Parses process output for structured messages and saves output as it is
666 read. Sets self.has_failure_output in case of evidence of a failure"""
667 for line_string
in output
.splitlines():
668 self
.process_line(line_string
)
670 if self
.saw_proc_start
and not self
.saw_proc_end
:
671 self
.has_failure_output
= True
673 def fix_text_output(self
, line
):
674 line
= cleanup_encoding(line
)
675 if self
.stack_fixer_function
is not None:
676 line
= self
.stack_fixer_function(line
)
678 if isinstance(line
, bytes
):
679 line
= line
.decode("utf-8")
682 def log_line(self
, line
):
683 """Log a line of output (either a parser json object or text output from
685 if isinstance(line
, six
.string_types
) or isinstance(line
, bytes
):
686 line
= self
.fix_text_output(line
).rstrip("\r\n")
687 self
.log
.process_output(self
.proc_ident
, line
, command
=self
.command
)
689 if "message" in line
:
690 line
["message"] = self
.fix_text_output(line
["message"])
691 if "xpcshell_process" in line
:
692 line
["thread"] = " ".join(
693 [current_thread().name
, line
["xpcshell_process"]]
696 line
["thread"] = current_thread().name
697 self
.log
.log_raw(line
)
699 def log_full_output(self
):
700 """Logs any buffered output from the test process, and clears the buffer."""
701 if not self
.output_lines
:
703 self
.log
.info(">>>>>>>")
704 for line
in self
.output_lines
:
706 self
.log
.info("<<<<<<<")
707 self
.output_lines
= []
709 def report_message(self
, message
):
710 """Stores or logs a json log message in mozlog format."""
712 self
.log_line(message
)
714 self
.output_lines
.append(message
)
716 def process_line(self
, line_string
):
717 """Parses a single line of output, determining its significance and
720 if isinstance(line_string
, bytes
):
721 # Transform binary to string representation
722 line_string
= line_string
.decode(sys
.stdout
.encoding
, errors
="replace")
724 if not line_string
.strip():
728 line_object
= json
.loads(line_string
)
729 if not isinstance(line_object
, dict):
730 self
.report_message(line_string
)
733 self
.report_message(line_string
)
737 "action" not in line_object
738 or line_object
["action"] not in EXPECTED_LOG_ACTIONS
740 # The test process output JSON.
741 self
.report_message(line_string
)
744 if line_object
["action"] == "crash_reporter_init":
745 self
.saw_crash_reporter_init
= True
748 action
= line_object
["action"]
750 self
.has_failure_output
= (
751 self
.has_failure_output
752 or "expected" in line_object
754 and line_object
["level"] == "ERROR"
757 self
.report_message(line_object
)
759 if action
== "log" and line_object
["message"] == "CHILD-TEST-STARTED":
760 self
.saw_proc_start
= True
761 elif action
== "log" and line_object
["message"] == "CHILD-TEST-COMPLETED":
762 self
.saw_proc_end
= True
765 """Run an individual xpcshell test."""
768 name
= self
.test_object
["id"]
769 path
= self
.test_object
["path"]
770 group
= get_full_group_name(self
.test_object
)
772 # Check for skipped tests
773 if "disabled" in self
.test_object
:
774 message
= self
.test_object
["disabled"]
776 message
= "disabled from xpcshell manifest"
777 self
.log
.test_start(name
, group
=group
)
778 self
.log
.test_end(name
, "SKIP", message
=message
, group
=group
)
781 self
.keep_going
= True
784 # Check for known-fail tests
785 expect_pass
= self
.test_object
["expected"] == "pass"
787 # By default self.appPath will equal the gre dir. If specified in the
788 # xpcshell.toml file, set a different app dir for this test.
789 if self
.app_dir_key
and self
.app_dir_key
in self
.test_object
:
790 rel_app_dir
= self
.test_object
[self
.app_dir_key
]
791 rel_app_dir
= os
.path
.join(self
.xrePath
, rel_app_dir
)
792 self
.appPath
= os
.path
.abspath(rel_app_dir
)
796 test_dir
= os
.path
.dirname(path
)
798 # Create a profile and a temp dir that the JS harness can stick
799 # a profile and temporary data in
800 self
.profileDir
= self
.setupProfileDir()
801 self
.tempDir
= self
.setupTempDir()
802 self
.mozInfoJSPath
= self
.setupMozinfoJS()
804 # Setup per-manifest prefs and write them into the tempdir.
805 self
.prefsFile
= self
.updateTestPrefsFile()
807 # The order of the command line is important:
808 # 1) Arguments for xpcshell itself
809 self
.command
= self
.buildXpcsCmd()
811 # 2) Arguments for the head files
812 self
.command
.extend(self
.buildCmdHead())
814 # 3) Arguments for the test file
815 self
.command
.extend(self
.buildCmdTestFile(path
))
816 self
.command
.extend(["-e", 'const _TEST_NAME = "%s";' % name
])
818 # 4) Arguments for code coverage
821 ["-e", 'const _JSCOV_DIR = "%s";' % self
.jscovdir
.replace("\\", "/")]
824 # 5) Runtime arguments
825 if "debug" in self
.test_object
:
826 self
.command
.append("-d")
828 self
.command
.extend(self
.xpcsRunArgs
)
830 if self
.test_object
.get("dmd") == "true":
831 self
.env
["PYTHON"] = sys
.executable
832 self
.env
["BREAKPAD_SYMBOLS_PATH"] = self
.symbolsPath
834 if self
.test_object
.get("snap") == "true":
835 self
.env
["SNAP_NAME"] = "firefox"
836 self
.env
["SNAP_INSTANCE_NAME"] = "firefox"
838 if self
.test_object
.get("subprocess") == "true":
839 self
.env
["PYTHON"] = sys
.executable
842 self
.test_object
.get("headless", "true" if self
.headless
else None)
845 self
.env
["MOZ_HEADLESS"] = "1"
846 self
.env
["DISPLAY"] = "77" # Set a fake display.
848 testTimeoutInterval
= self
.harness_timeout
849 # Allow a test to request a multiple of the timeout if it is expected to take long
850 if "requesttimeoutfactor" in self
.test_object
:
851 testTimeoutInterval
*= int(self
.test_object
["requesttimeoutfactor"])
854 if not self
.interactive
and not self
.debuggerInfo
and not self
.jsDebuggerInfo
:
855 testTimer
= Timer(testTimeoutInterval
, lambda: self
.testTimeout(proc
))
859 process_output
= None
862 self
.log
.test_start(name
, group
=group
)
864 self
.logCommand(name
, self
.command
, test_dir
)
866 proc
= self
.launchProcess(
872 timeout
=testTimeoutInterval
,
876 if hasattr(proc
, "pid"):
877 self
.proc_ident
= proc
.pid
879 # On mobile, "proc" is just a file.
880 self
.proc_ident
= name
883 self
.log
.info("%s | Process ID: %d" % (name
, self
.proc_ident
))
885 # Communicate returns a tuple of (stdout, stderr), however we always
886 # redirect stderr to stdout, so the second element is ignored.
887 process_output
, _
= self
.communicate(proc
)
890 # Not sure what else to do here...
891 self
.keep_going
= True
898 # For the remote case, stdout is not yet depleted, so we parse
899 # it here all at once.
900 self
.parse_output(process_output
)
902 return_code
= self
.getReturnCode(proc
)
904 # TSan'd processes return 66 if races are detected. This isn't
905 # good in the sense that there's no way to distinguish between
906 # a process that would normally have returned zero but has races,
907 # and a race-free process that returns 66. But I don't see how
908 # to do better. This ambiguity is at least constrained to the
909 # with-TSan case. It doesn't affect normal builds.
911 # This also assumes that the magic value 66 isn't overridden by
912 # a TSAN_OPTIONS=exitcode=<number> environment variable setting.
914 TSAN_EXIT_CODE_WITH_RACES
= 66
916 return_code_ok
= return_code
== 0 or (
917 self
.usingTSan
and return_code
== TSAN_EXIT_CODE_WITH_RACES
920 # Due to the limitation on the remote xpcshell test, the process
921 # return code does not represent the process crash.
922 # If crash_reporter_init log has not been seen and the return code
923 # is 0, it means the process crashed before setting up the crash
926 # NOTE: Crash reporter is not enabled on some configuration, such
927 # as ASAN and TSAN. Those configuration shouldn't be using
928 # remote xpcshell test, and the crash should be caught by
929 # the process return code.
930 # NOTE: self.saw_crash_reporter_init is False also when adb failed
931 # to launch process, and in that case the return code is
933 # (see launchProcess in remotexpcshelltests.py)
934 ended_before_crash_reporter_init
= (
936 and self
.usingCrashReporter
937 and not self
.saw_crash_reporter_init
941 (not self
.has_failure_output
)
942 and not ended_before_crash_reporter_init
946 status
= "PASS" if passed
else "FAIL"
947 expected
= "PASS" if expect_pass
else "FAIL"
948 message
= "xpcshell return code: %d" % return_code
953 if status
!= expected
or ended_before_crash_reporter_init
:
954 if ended_before_crash_reporter_init
:
959 message
="Test ended before setting up the crash reporter",
967 message
="Test failed or timed out, will retry",
970 self
.clean_temp_dirs(path
)
971 if self
.verboseIfFails
and not self
.verbose
:
972 self
.log_full_output()
976 name
, status
, expected
=expected
, message
=message
, group
=group
978 self
.log_full_output()
982 if self
.failureManifest
:
983 with
open(self
.failureManifest
, "a") as f
:
984 f
.write("[%s]\n" % self
.test_object
["path"])
985 for k
, v
in self
.test_object
.items():
986 f
.write("%s = %s\n" % (k
, v
))
989 # If TSan reports a race, dump the output, else we can't
990 # diagnose what the problem was. See comments above about
991 # the significance of TSAN_EXIT_CODE_WITH_RACES.
992 if self
.usingTSan
and return_code
== TSAN_EXIT_CODE_WITH_RACES
:
993 self
.log_full_output()
996 name
, status
, expected
=expected
, message
=message
, group
=group
999 self
.log_full_output()
1008 if self
.checkForCrashes(self
.tempDir
, self
.symbolsPath
, test_name
=name
):
1010 self
.clean_temp_dirs(path
)
1013 # If we assert during shutdown there's a chance the test has passed
1014 # but we haven't logged full output, so do so here.
1015 self
.log_full_output()
1018 if self
.logfiles
and process_output
:
1019 self
.createLogFile(name
, process_output
)
1022 self
.postCheck(proc
)
1023 self
.clean_temp_dirs(path
)
1026 self
.log
.error("Received SIGINT (control-C) during test execution")
1030 self
.keep_going
= False
1033 self
.keep_going
= True
1036 class XPCShellTests(object):
1037 def __init__(self
, log
=None):
1038 """Initializes node status and logger."""
1040 self
.harness_timeout
= HARNESS_TIMEOUT
1042 self
.http3Server
= None
1043 self
.conditioned_profile_dir
= None
1045 def getTestManifest(self
, manifest
):
1046 if isinstance(manifest
, TestManifest
):
1048 elif manifest
is not None:
1049 manifest
= os
.path
.normpath(os
.path
.abspath(manifest
))
1050 if os
.path
.isfile(manifest
):
1051 return TestManifest([manifest
], strict
=True)
1053 toml_path
= os
.path
.join(manifest
, "xpcshell.toml")
1055 toml_path
= os
.path
.join(SCRIPT_DIR
, "tests", "xpcshell.toml")
1057 if os
.path
.exists(toml_path
):
1058 return TestManifest([toml_path
], strict
=True)
1061 "Failed to find manifest at %s; use --manifest "
1062 "to set path explicitly." % toml_path
1066 def normalizeTest(self
, root
, test_object
):
1067 path
= test_object
.get("file_relpath", test_object
["relpath"])
1068 if "dupe-manifest" in test_object
and "ancestor_manifest" in test_object
:
1069 test_object
["id"] = "%s:%s" % (
1070 os
.path
.basename(test_object
["ancestor_manifest"]),
1074 test_object
["id"] = path
1077 test_object
["manifest"] = os
.path
.relpath(test_object
["manifest"], root
)
1080 for key
in ("id", "manifest"):
1081 test_object
[key
] = test_object
[key
].replace(os
.sep
, "/")
1085 def buildTestList(self
, test_tags
=None, test_paths
=None, verify
=False):
1086 """Reads the xpcshell.toml manifest and set self.alltests to an array.
1088 Given the parameters, this method compiles a list of tests to be run
1089 that matches the criteria set by parameters.
1091 If any chunking of tests are to occur, it is also done in this method.
1093 If no tests are added to the list of tests to be run, an error
1094 is logged. A sys.exit() signal is sent to the caller.
1097 test_tags (list, optional): list of strings.
1098 test_paths (list, optional): list of strings derived from the command
1099 line argument provided by user, specifying
1101 verify (bool, optional): boolean value.
1103 if test_paths
is None:
1106 mp
= self
.getTestManifest(self
.manifest
)
1109 if build
and not root
:
1110 root
= build
.topsrcdir
1111 normalize
= partial(self
.normalizeTest
, root
)
1115 filters
.append(tags(test_tags
))
1119 path_filter
= pathprefix(test_paths
)
1120 filters
.append(path_filter
)
1122 noDefaultFilters
= False
1123 if self
.runFailures
:
1124 filters
.append(failures(self
.runFailures
))
1125 noDefaultFilters
= True
1127 if self
.totalChunks
> 1:
1128 filters
.append(chunk_by_slice(self
.thisChunk
, self
.totalChunks
))
1130 self
.alltests
= list(
1135 noDefaultFilters
=noDefaultFilters
,
1141 sys
.stderr
.write("*** offending mozinfo.info: %s\n" % repr(mozinfo
.info
))
1144 if path_filter
and path_filter
.missing
:
1146 "The following path(s) didn't resolve any tests:\n {}".format(
1147 " \n".join(sorted(path_filter
.missing
))
1151 if len(self
.alltests
) == 0:
1154 and path_filter
.missing
== set(test_paths
)
1155 and os
.environ
.get("MOZ_AUTOMATION") == "1"
1157 # This can happen in CI when a manifest doesn't exist due to a
1158 # build config variable in moz.build traversal. Don't generate
1159 # an error in this case. Adding a todo count avoids mozharness
1161 self
.todoCount
+= len(path_filter
.missing
)
1164 "no tests to run using specified "
1165 "combination of filters: {}".format(mp
.fmt_filters())
1169 if len(self
.alltests
) == 1 and not verify
:
1170 self
.singleFile
= os
.path
.basename(self
.alltests
[0]["path"])
1172 self
.singleFile
= None
1175 self
.dump_tests
= os
.path
.expanduser(self
.dump_tests
)
1176 assert os
.path
.exists(os
.path
.dirname(self
.dump_tests
))
1177 with
open(self
.dump_tests
, "w") as dumpFile
:
1178 dumpFile
.write(json
.dumps({"active_tests": self
.alltests
}))
1180 self
.log
.info("Dumping active_tests to %s file." % self
.dump_tests
)
1183 def setAbsPath(self
):
1185 Set the absolute path for xpcshell and xrepath. These 3 variables
1186 depend on input from the command line and we need to allow for absolute paths.
1187 This function is overloaded for a remote solution as os.path* won't work remotely.
1189 self
.testharnessdir
= os
.path
.dirname(os
.path
.abspath(__file__
))
1190 self
.headJSPath
= self
.testharnessdir
.replace("\\", "/") + "/head.js"
1191 if self
.xpcshell
is not None:
1192 self
.xpcshell
= os
.path
.abspath(self
.xpcshell
)
1194 if self
.app_binary
is not None:
1195 self
.app_binary
= os
.path
.abspath(self
.app_binary
)
1197 if self
.xrePath
is None:
1198 binary_path
= self
.app_binary
or self
.xpcshell
1199 self
.xrePath
= os
.path
.dirname(binary_path
)
1201 # Check if we're run from an OSX app bundle and override
1202 # self.xrePath if we are.
1203 appBundlePath
= os
.path
.join(
1204 os
.path
.dirname(os
.path
.dirname(self
.xpcshell
)), "Resources"
1206 if os
.path
.exists(os
.path
.join(appBundlePath
, "application.ini")):
1207 self
.xrePath
= appBundlePath
1209 self
.xrePath
= os
.path
.abspath(self
.xrePath
)
1211 if self
.mozInfo
is None:
1212 self
.mozInfo
= os
.path
.join(self
.testharnessdir
, "mozinfo.json")
1214 def buildPrefsFile(self
, extraPrefs
):
1215 # Create the prefs.js file
1217 # In test packages used in CI, the profile_data directory is installed
1218 # in the SCRIPT_DIR.
1219 profile_data_dir
= os
.path
.join(SCRIPT_DIR
, "profile_data")
1220 # If possible, read profile data from topsrcdir. This prevents us from
1221 # requiring a re-build to pick up newly added extensions in the
1222 # <profile>/extensions directory.
1224 path
= os
.path
.join(build
.topsrcdir
, "testing", "profiles")
1225 if os
.path
.isdir(path
):
1226 profile_data_dir
= path
1227 # Still not found? Look for testing/profiles relative to testing/xpcshell.
1228 if not os
.path
.isdir(profile_data_dir
):
1229 path
= os
.path
.abspath(os
.path
.join(SCRIPT_DIR
, "..", "profiles"))
1230 if os
.path
.isdir(path
):
1231 profile_data_dir
= path
1233 with
open(os
.path
.join(profile_data_dir
, "profiles.json"), "r") as fh
:
1234 base_profiles
= json
.load(fh
)["xpcshell"]
1236 # values to use when interpolating preferences
1238 "server": "dummyserver",
1241 profile
= Profile(profile
=self
.tempDir
, restore
=False)
1242 prefsFile
= os
.path
.join(profile
.profile
, "user.js")
1244 # Empty the user.js file in case the file existed before.
1245 with
open(prefsFile
, "w"):
1248 for name
in base_profiles
:
1249 path
= os
.path
.join(profile_data_dir
, name
)
1250 profile
.merge(path
, interpolation
=interpolation
)
1252 # add command line prefs
1253 prefs
= parse_preferences(extraPrefs
)
1254 profile
.set_preferences(prefs
)
1256 self
.prefsFile
= prefsFile
1259 def buildCoreEnvironment(self
):
1261 Add environment variables likely to be used across all platforms, including
1264 # Make assertions fatal
1265 self
.env
["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
1266 # Crash reporting interferes with debugging
1267 if not self
.debuggerInfo
:
1268 self
.env
["MOZ_CRASHREPORTER"] = "1"
1269 # Don't launch the crash reporter client
1270 self
.env
["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
1271 # Don't permit remote connections by default.
1272 # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
1273 # enable non-local connections for the purposes of local testing.
1274 # Don't override the user's choice here. See bug 1049688.
1275 self
.env
.setdefault("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "1")
1276 if self
.mozInfo
.get("topsrcdir") is not None:
1277 self
.env
["MOZ_DEVELOPER_REPO_DIR"] = self
.mozInfo
["topsrcdir"]
1278 if self
.mozInfo
.get("topobjdir") is not None:
1279 self
.env
["MOZ_DEVELOPER_OBJ_DIR"] = self
.mozInfo
["topobjdir"]
1281 # Disable the content process sandbox for the xpcshell tests. They
1282 # currently attempt to do things like bind() sockets, which is not
1283 # compatible with the sandbox.
1284 self
.env
["MOZ_DISABLE_CONTENT_SANDBOX"] = "1"
1285 if os
.getenv("MOZ_FETCHES_DIR", None):
1286 self
.env
["MOZ_FETCHES_DIR"] = os
.getenv("MOZ_FETCHES_DIR", None)
1288 if self
.mozInfo
.get("socketprocess_networking"):
1289 self
.env
["MOZ_FORCE_USE_SOCKET_PROCESS"] = "1"
1291 self
.env
["MOZ_DISABLE_SOCKET_PROCESS"] = "1"
1293 def buildEnvironment(self
):
1295 Create and returns a dictionary of self.env to include all the appropriate env
1296 variables and values. On a remote system, we overload this to set different
1297 values and are missing things like os.environ and PATH.
1299 self
.env
= dict(os
.environ
)
1300 self
.buildCoreEnvironment()
1301 if sys
.platform
== "win32":
1302 self
.env
["PATH"] = self
.env
["PATH"] + ";" + self
.xrePath
1303 elif sys
.platform
in ("os2emx", "os2knix"):
1304 os
.environ
["BEGINLIBPATH"] = self
.xrePath
+ ";" + self
.env
["BEGINLIBPATH"]
1305 os
.environ
["LIBPATHSTRICT"] = "T"
1306 elif sys
.platform
== "osx" or sys
.platform
== "darwin":
1307 self
.env
["DYLD_LIBRARY_PATH"] = os
.path
.join(
1308 os
.path
.dirname(self
.xrePath
), "MacOS"
1310 else: # unix or linux?
1311 if "LD_LIBRARY_PATH" not in self
.env
or self
.env
["LD_LIBRARY_PATH"] is None:
1312 self
.env
["LD_LIBRARY_PATH"] = self
.xrePath
1314 self
.env
["LD_LIBRARY_PATH"] = ":".join(
1315 [self
.xrePath
, self
.env
["LD_LIBRARY_PATH"]]
1318 usingASan
= "asan" in self
.mozInfo
and self
.mozInfo
["asan"]
1319 usingTSan
= "tsan" in self
.mozInfo
and self
.mozInfo
["tsan"]
1320 if usingASan
or usingTSan
:
1321 # symbolizer support
1322 if "ASAN_SYMBOLIZER_PATH" in self
.env
and os
.path
.isfile(
1323 self
.env
["ASAN_SYMBOLIZER_PATH"]
1325 llvmsym
= self
.env
["ASAN_SYMBOLIZER_PATH"]
1327 llvmsym
= os
.path
.join(
1328 self
.xrePath
, "llvm-symbolizer" + self
.mozInfo
["bin_suffix"]
1330 if os
.path
.isfile(llvmsym
):
1332 self
.env
["ASAN_SYMBOLIZER_PATH"] = llvmsym
1334 oldTSanOptions
= self
.env
.get("TSAN_OPTIONS", "")
1335 self
.env
["TSAN_OPTIONS"] = "external_symbolizer_path={} {}".format(
1336 llvmsym
, oldTSanOptions
1338 self
.log
.info("runxpcshelltests.py | using symbolizer at %s" % llvmsym
)
1341 "TEST-UNEXPECTED-FAIL | runxpcshelltests.py | "
1342 "Failed to find symbolizer at %s" % llvmsym
1349 Determine the value of the stdout and stderr for the test.
1350 Return value is a list (pStdout, pStderr).
1352 if self
.interactive
:
1356 if self
.debuggerInfo
and self
.debuggerInfo
.interactive
:
1360 if sys
.platform
== "os2emx":
1365 return pStdout
, pStderr
1367 def verifyDirPath(self
, dirname
):
1369 Simple wrapper to get the absolute path for a given directory name.
1370 On a remote system, we need to overload this to work on the remote filesystem.
1372 return os
.path
.abspath(dirname
)
1374 def trySetupNode(self
):
1376 Run node for HTTP/2 tests, if available, and updates mozinfo as appropriate.
1378 if os
.getenv("MOZ_ASSUME_NODE_RUNNING", None):
1379 self
.log
.info("Assuming required node servers are already running")
1380 if not os
.getenv("MOZHTTP2_PORT", None):
1382 "MOZHTTP2_PORT environment variable not set. "
1383 "Tests requiring http/2 will fail."
1387 # We try to find the node executable in the path given to us by the user in
1388 # the MOZ_NODE_PATH environment variable
1389 nodeBin
= os
.getenv("MOZ_NODE_PATH", None)
1390 if not nodeBin
and build
:
1391 nodeBin
= build
.substs
.get("NODEJS")
1394 "MOZ_NODE_PATH environment variable not set. "
1395 "Tests requiring http/2 will fail."
1399 if not os
.path
.exists(nodeBin
) or not os
.path
.isfile(nodeBin
):
1400 error
= "node not found at MOZ_NODE_PATH %s" % (nodeBin
)
1401 self
.log
.error(error
)
1402 raise IOError(error
)
1404 self
.log
.info("Found node at %s" % (nodeBin
,))
1406 def read_streams(name
, proc
, pipe
):
1407 output
= "stdout" if pipe
== proc
.stdout
else "stderr"
1408 for line
in iter(pipe
.readline
, ""):
1409 self
.log
.info("node %s [%s] %s" % (name
, output
, line
))
1411 def startServer(name
, serverJs
):
1412 if not os
.path
.exists(serverJs
):
1413 error
= "%s not found at %s" % (name
, serverJs
)
1414 self
.log
.error(error
)
1415 raise IOError(error
)
1417 # OK, we found our server, let's try to get it running
1418 self
.log
.info("Found %s at %s" % (name
, serverJs
))
1420 # We pipe stdin to node because the server will exit when its
1422 with
popenCleanupHack():
1424 [nodeBin
, serverJs
],
1430 universal_newlines
=True,
1431 start_new_session
=True,
1433 self
.nodeProc
[name
] = process
1435 # Check to make sure the server starts properly by waiting for it to
1436 # tell us it's started
1437 msg
= process
.stdout
.readline()
1438 if "server listening" in msg
:
1439 searchObj
= re
.search(
1440 r
"HTTP2 server listening on ports ([0-9]+),([0-9]+)", msg
, 0
1443 self
.env
["MOZHTTP2_PORT"] = searchObj
.group(1)
1444 self
.env
["MOZNODE_EXEC_PORT"] = searchObj
.group(2)
1446 target
=read_streams
,
1447 args
=(name
, process
, process
.stdout
),
1452 target
=read_streams
,
1453 args
=(name
, process
, process
.stderr
),
1457 except OSError as e
:
1458 # This occurs if the subprocess couldn't be started
1459 self
.log
.error("Could not run %s server: %s" % (name
, str(e
)))
1462 myDir
= os
.path
.split(os
.path
.abspath(__file__
))[0]
1463 startServer("moz-http2", os
.path
.join(myDir
, "moz-http2", "moz-http2.js"))
1465 def shutdownNode(self
):
1467 Shut down our node process, if it exists
1469 for name
, proc
in six
.iteritems(self
.nodeProc
):
1470 self
.log
.info("Node %s server shutting down ..." % name
)
1471 if proc
.poll() is not None:
1472 self
.log
.info("Node server %s already dead %s" % (name
, proc
.poll()))
1473 elif sys
.platform
!= "win32":
1474 # Kill process and all its spawned children.
1475 os
.killpg(proc
.pid
, signal
.SIGTERM
)
1481 def startHttp3Server(self
):
1483 Start a Http3 test server.
1486 if sys
.platform
== "win32":
1488 http3ServerPath
= self
.http3ServerPath
1489 if not http3ServerPath
:
1490 http3ServerPath
= os
.path
.join(
1491 SCRIPT_DIR
, "http3server", "http3server" + binSuffix
1494 http3ServerPath
= os
.path
.join(
1495 build
.topobjdir
, "dist", "bin", "http3server" + binSuffix
1497 dbPath
= os
.path
.join(SCRIPT_DIR
, "http3server", "http3serverDB")
1499 dbPath
= os
.path
.join(build
.topsrcdir
, "netwerk", "test", "http3serverDB")
1501 options
["http3ServerPath"] = http3ServerPath
1502 options
["profilePath"] = dbPath
1503 options
["isMochitest"] = False
1504 options
["isWin"] = sys
.platform
== "win32"
1505 serverEnv
= self
.env
.copy()
1506 serverLog
= self
.env
.get("MOZHTTP3_SERVER_LOG")
1507 if serverLog
is not None:
1508 serverEnv
["RUST_LOG"] = serverLog
1509 self
.http3Server
= Http3Server(options
, serverEnv
, self
.log
)
1510 self
.http3Server
.start()
1511 for key
, value
in self
.http3Server
.ports().items():
1512 self
.env
[key
] = value
1513 self
.env
["MOZHTTP3_ECH"] = self
.http3Server
.echConfig()
1515 def shutdownHttp3Server(self
):
1516 if self
.http3Server
is None:
1518 self
.http3Server
.stop()
1519 self
.http3Server
= None
1521 def buildXpcsRunArgs(self
):
1523 Add arguments to run the test or make it interactive.
1525 if self
.interactive
:
1526 self
.xpcsRunArgs
= [
1528 'print("To start the test, type |_execute_test();|.");',
1532 self
.xpcsRunArgs
= ["-e", "_execute_test(); quit(0);"]
1534 def addTestResults(self
, test
):
1535 self
.passCount
+= test
.passCount
1536 self
.failCount
+= test
.failCount
1537 self
.todoCount
+= test
.todoCount
1539 def updateMozinfo(self
, prefs
, options
):
1540 # Handle filenames in mozInfo
1541 if not isinstance(self
.mozInfo
, dict):
1542 mozInfoFile
= self
.mozInfo
1543 if not os
.path
.isfile(mozInfoFile
):
1545 "Error: couldn't find mozinfo.json at '%s'. Perhaps you "
1546 "need to use --build-info-json?" % mozInfoFile
1549 self
.mozInfo
= json
.load(open(mozInfoFile
))
1551 # mozinfo.info is used as kwargs. Some builds are done with
1552 # an older Python that can't handle Unicode keys in kwargs.
1553 # All of the keys in question should be ASCII.
1555 for k
, v
in self
.mozInfo
.items():
1556 if isinstance(k
, bytes
):
1557 k
= k
.decode("utf-8")
1559 self
.mozInfo
= fixedInfo
1561 self
.mozInfo
["fission"] = prefs
.get("fission.autostart", True)
1562 self
.mozInfo
["sessionHistoryInParent"] = self
.mozInfo
[
1564 ] or not prefs
.get("fission.disableSessionHistoryInParent", False)
1566 self
.mozInfo
["serviceworker_e10s"] = True
1568 self
.mozInfo
["verify"] = options
.get("verify", False)
1570 self
.mozInfo
["socketprocess_networking"] = prefs
.get(
1571 "network.http.network_access_on_socket_process.enabled", False
1574 self
.mozInfo
["condprof"] = options
.get("conditionedProfile", False)
1576 self
.mozInfo
["msix"] = options
.get(
1578 ) is not None and "WindowsApps" in options
.get("app_binary", "")
1580 self
.mozInfo
["is_ubuntu"] = "Ubuntu" in platform
.version()
1582 mozinfo
.update(self
.mozInfo
)
1587 def conditioned_profile_copy(self
):
1588 """Returns a copy of the original conditioned profile that was created."""
1589 condprof_copy
= os
.path
.join(tempfile
.mkdtemp(), "profile")
1591 self
.conditioned_profile_dir
,
1593 ignore
=shutil
.ignore_patterns("lock"),
1595 self
.log
.info("Created a conditioned-profile copy: %s" % condprof_copy
)
1596 return condprof_copy
1598 def downloadConditionedProfile(self
, profile_scenario
, app
):
1599 from condprof
.client
import get_profile
1600 from condprof
.util
import get_current_platform
, get_version
1602 if self
.conditioned_profile_dir
:
1603 # We already have a directory, so provide a copy that
1604 # will get deleted after it's done with
1605 return self
.conditioned_profile_dir
1607 # create a temp file to help ensure uniqueness
1608 temp_download_dir
= tempfile
.mkdtemp()
1610 "Making temp_download_dir from inside get_conditioned_profile {}".format(
1614 # call condprof's client API to yield our platform-specific
1615 # conditioned-profile binary
1616 platform
= get_current_platform()
1618 if isinstance(app
, str):
1619 version
= get_version(app
)
1621 if not profile_scenario
:
1622 profile_scenario
= "settled"
1624 cond_prof_target_dir
= get_profile(
1628 repo
="mozilla-central",
1634 # any other error is a showstopper
1635 self
.log
.critical("Could not get the conditioned profile")
1636 traceback
.print_exc()
1640 self
.log
.info("Retrying a profile with no version specified")
1641 cond_prof_target_dir
= get_profile(
1645 repo
="mozilla-central",
1649 self
.log
.critical("Could not get the conditioned profile")
1650 traceback
.print_exc()
1653 # now get the full directory path to our fetched conditioned profile
1654 self
.conditioned_profile_dir
= os
.path
.join(
1655 temp_download_dir
, cond_prof_target_dir
1657 if not os
.path
.exists(cond_prof_target_dir
):
1659 "Can't find target_dir {}, from get_profile()"
1660 "temp_download_dir {}, platform {}, scenario {}".format(
1661 cond_prof_target_dir
, temp_download_dir
, platform
, profile_scenario
1667 "Original self.conditioned_profile_dir is now set: {}".format(
1668 self
.conditioned_profile_dir
1671 return self
.conditioned_profile_copy
1673 def runSelfTest(self
):
1680 class XPCShellTestsTests(selftest
.XPCShellTestsTests
):
1681 def __init__(self
, name
):
1682 unittest
.TestCase
.__init
__(self
, name
)
1683 self
.testing_modules
= this
.testingModulesDir
1684 self
.xpcshellBin
= this
.xpcshell
1685 self
.app_binary
= this
.app_binary
1686 self
.utility_path
= this
.utility_path
1687 self
.symbols_path
= this
.symbolsPath
1689 old_info
= dict(mozinfo
.info
)
1691 suite
= unittest
.TestLoader().loadTestsFromTestCase(XPCShellTestsTests
)
1692 return unittest
.TextTestRunner(verbosity
=2).run(suite
).wasSuccessful()
1694 # The self tests modify mozinfo, so we need to reset it.
1695 mozinfo
.info
.clear()
1696 mozinfo
.update(old_info
)
1698 def runTests(self
, options
, testClass
=XPCShellTestThread
, mobileArgs
=None):
1704 # Number of times to repeat test(s) in --verify mode
1707 if isinstance(options
, Namespace
):
1708 options
= vars(options
)
1710 # Try to guess modules directory.
1711 # This somewhat grotesque hack allows the buildbot machines to find the
1712 # modules directory without having to configure the buildbot hosts. This
1713 # code path should never be executed in local runs because the build system
1714 # should always set this argument.
1715 if not options
.get("testingModulesDir"):
1716 possible
= os
.path
.join(here
, os
.path
.pardir
, "modules")
1718 if os
.path
.isdir(possible
):
1719 testingModulesDir
= possible
1721 if options
.get("rerun_failures"):
1722 if os
.path
.exists(options
.get("failure_manifest")):
1723 rerun_manifest
= os
.path
.join(
1724 os
.path
.dirname(options
["failure_manifest"]), "rerun.toml"
1726 shutil
.copyfile(options
["failure_manifest"], rerun_manifest
)
1727 os
.remove(options
["failure_manifest"])
1729 self
.log
.error("No failures were found to re-run.")
1732 if options
.get("testingModulesDir"):
1733 # The resource loader expects native paths. Depending on how we were
1734 # invoked, a UNIX style path may sneak in on Windows. We try to
1736 testingModulesDir
= os
.path
.normpath(options
["testingModulesDir"])
1738 if not os
.path
.isabs(testingModulesDir
):
1739 testingModulesDir
= os
.path
.abspath(testingModulesDir
)
1741 if not testingModulesDir
.endswith(os
.path
.sep
):
1742 testingModulesDir
+= os
.path
.sep
1744 self
.debuggerInfo
= None
1746 if options
.get("debugger"):
1747 self
.debuggerInfo
= mozdebug
.get_debugger_info(
1748 options
.get("debugger"),
1749 options
.get("debuggerArgs"),
1750 options
.get("debuggerInteractive"),
1753 self
.jsDebuggerInfo
= None
1754 if options
.get("jsDebugger"):
1755 # A namedtuple let's us keep .port instead of ['port']
1756 JSDebuggerInfo
= namedtuple("JSDebuggerInfo", ["port"])
1757 self
.jsDebuggerInfo
= JSDebuggerInfo(port
=options
["jsDebuggerPort"])
1759 self
.app_binary
= options
.get("app_binary")
1760 self
.xpcshell
= options
.get("xpcshell")
1761 self
.http3ServerPath
= options
.get("http3server")
1762 self
.xrePath
= options
.get("xrePath")
1763 self
.utility_path
= options
.get("utility_path")
1764 self
.appPath
= options
.get("appPath")
1765 self
.symbolsPath
= options
.get("symbolsPath")
1766 self
.tempDir
= os
.path
.normpath(options
.get("tempDir") or tempfile
.gettempdir())
1767 self
.manifest
= options
.get("manifest")
1768 self
.dump_tests
= options
.get("dump_tests")
1769 self
.interactive
= options
.get("interactive")
1770 self
.verbose
= options
.get("verbose")
1771 self
.verboseIfFails
= options
.get("verboseIfFails")
1772 self
.keepGoing
= options
.get("keepGoing")
1773 self
.logfiles
= options
.get("logfiles")
1774 self
.totalChunks
= options
.get("totalChunks", 1)
1775 self
.thisChunk
= options
.get("thisChunk")
1776 self
.profileName
= options
.get("profileName") or "xpcshell"
1777 self
.mozInfo
= options
.get("mozInfo")
1778 self
.testingModulesDir
= testingModulesDir
1779 self
.sequential
= options
.get("sequential")
1780 self
.failure_manifest
= options
.get("failure_manifest")
1781 self
.threadCount
= options
.get("threadCount") or NUM_THREADS
1782 self
.jscovdir
= options
.get("jscovdir")
1783 self
.headless
= options
.get("headless")
1784 self
.runFailures
= options
.get("runFailures")
1785 self
.timeoutAsPass
= options
.get("timeoutAsPass")
1786 self
.crashAsPass
= options
.get("crashAsPass")
1787 self
.conditionedProfile
= options
.get("conditionedProfile")
1788 self
.repeat
= options
.get("repeat", 0)
1795 if self
.conditionedProfile
:
1796 self
.conditioned_profile_dir
= self
.downloadConditionedProfile(
1797 "full", self
.appPath
1799 options
["self_test"] = False
1800 if not options
["test_tags"]:
1801 options
["test_tags"] = []
1802 options
["test_tags"].append("condprof")
1806 eprefs
= options
.get("extraPrefs") or []
1807 # enable fission by default
1808 if options
.get("disableFission"):
1809 eprefs
.append("fission.autostart=false")
1811 # should be by default, just in case
1812 eprefs
.append("fission.autostart=true")
1814 prefs
= self
.buildPrefsFile(eprefs
)
1815 self
.buildXpcsRunArgs()
1817 self
.event
= Event()
1819 if not self
.updateMozinfo(prefs
, options
):
1823 "These variables are available in the mozinfo environment and "
1824 "can be used to skip tests conditionally:"
1826 for info
in sorted(self
.mozInfo
.items(), key
=lambda item
: item
[0]):
1827 self
.log
.info(" {key}: {value}".format(key
=info
[0], value
=info
[1]))
1829 if options
.get("self_test"):
1830 if not self
.runSelfTest():
1834 ("tsan" in self
.mozInfo
and self
.mozInfo
["tsan"])
1835 or ("asan" in self
.mozInfo
and self
.mozInfo
["asan"])
1836 ) and not options
.get("threadCount"):
1837 # TSan/ASan require significantly more memory, so reduce the amount of parallel
1838 # tests we run to avoid OOMs and timeouts. We always keep a minimum of 2 for
1839 # non-sequential execution.
1840 # pylint --py3k W1619
1841 self
.threadCount
= max(self
.threadCount
/ 2, 2)
1843 self
.stack_fixer_function
= None
1844 if self
.utility_path
and os
.path
.exists(self
.utility_path
):
1845 self
.stack_fixer_function
= get_stack_fixer_function(
1846 self
.utility_path
, self
.symbolsPath
1849 # buildEnvironment() needs mozInfo, so we call it after mozInfo is initialized.
1850 self
.buildEnvironment()
1852 # The appDirKey is a optional entry in either the default or individual test
1853 # sections that defines a relative application directory for test runs. If
1854 # defined we pass 'grePath/$appDirKey' for the -a parameter of the xpcshell
1857 if "appname" in self
.mozInfo
:
1858 appDirKey
= self
.mozInfo
["appname"] + "-appdir"
1860 # We have to do this before we run tests that depend on having the node
1864 self
.startHttp3Server()
1866 pStdout
, pStderr
= self
.getPipes()
1869 options
.get("test_tags"), options
.get("testPaths"), options
.get("verify")
1872 self
.sequential
= True
1874 if options
.get("shuffle"):
1875 random
.shuffle(self
.alltests
)
1877 self
.cleanup_dir_list
= []
1880 "appPath": self
.appPath
,
1881 "xrePath": self
.xrePath
,
1882 "utility_path": self
.utility_path
,
1883 "testingModulesDir": self
.testingModulesDir
,
1884 "debuggerInfo": self
.debuggerInfo
,
1885 "jsDebuggerInfo": self
.jsDebuggerInfo
,
1886 "headJSPath": self
.headJSPath
,
1887 "tempDir": self
.tempDir
,
1888 "testharnessdir": self
.testharnessdir
,
1889 "profileName": self
.profileName
,
1890 "singleFile": self
.singleFile
,
1891 "env": self
.env
, # making a copy of this in the testthreads
1892 "symbolsPath": self
.symbolsPath
,
1893 "logfiles": self
.logfiles
,
1894 "app_binary": self
.app_binary
,
1895 "xpcshell": self
.xpcshell
,
1896 "xpcsRunArgs": self
.xpcsRunArgs
,
1897 "failureManifest": self
.failure_manifest
,
1898 "jscovdir": self
.jscovdir
,
1899 "harness_timeout": self
.harness_timeout
,
1900 "stack_fixer_function": self
.stack_fixer_function
,
1901 "event": self
.event
,
1902 "cleanup_dir_list": self
.cleanup_dir_list
,
1905 "keep_going": self
.keepGoing
,
1907 "interactive": self
.interactive
,
1908 "app_dir_key": appDirKey
,
1909 "rootPrefsFile": self
.prefsFile
,
1910 "extraPrefs": options
.get("extraPrefs") or [],
1911 "verboseIfFails": self
.verboseIfFails
,
1912 "headless": self
.headless
,
1913 "runFailures": self
.runFailures
,
1914 "timeoutAsPass": self
.timeoutAsPass
,
1915 "crashAsPass": self
.crashAsPass
,
1916 "conditionedProfileDir": self
.conditioned_profile_dir
,
1917 "repeat": self
.repeat
,
1921 # Allow user to kill hung xpcshell subprocess with SIGINT
1922 # when we are only running tests sequentially.
1923 signal
.signal(signal
.SIGINT
, markGotSIGINT
)
1925 if self
.debuggerInfo
:
1926 # Force a sequential run
1927 self
.sequential
= True
1929 # If we have an interactive debugger, disable SIGINT entirely.
1930 if self
.debuggerInfo
.interactive
:
1931 signal
.signal(signal
.SIGINT
, lambda signum
, frame
: None)
1933 if "lldb" in self
.debuggerInfo
.path
:
1934 # Ask people to start debugging using 'process launch', see bug 952211.
1936 "It appears that you're using LLDB to debug this test. "
1937 + "Please use the 'process launch' command instead of "
1938 "the 'run' command to start xpcshell."
1941 if self
.jsDebuggerInfo
:
1942 # The js debugger magic needs more work to do the right thing
1943 # if debugging multiple files.
1944 if len(self
.alltests
) != 1:
1946 "Error: --jsdebugger can only be used with a single test!"
1950 # The test itself needs to know whether it is a tsan build, since
1951 # that has an effect on interpretation of the process return value.
1952 usingTSan
= "tsan" in self
.mozInfo
and self
.mozInfo
["tsan"]
1954 usingCrashReporter
= (
1955 "crashreporter" in self
.mozInfo
and self
.mozInfo
["crashreporter"]
1958 # create a queue of all tests that will run
1959 tests_queue
= deque()
1960 # also a list for the tests that need to be run sequentially
1961 sequential_tests
= []
1964 if options
.get("repeat", 0) > 0:
1965 self
.sequential
= True
1967 if not options
.get("verify"):
1968 for test_object
in self
.alltests
:
1969 # Test identifiers are provided for the convenience of logging. These
1970 # start as path names but are rewritten in case tests from the same path
1973 path
= test_object
["path"]
1975 if self
.singleFile
and not path
.endswith(self
.singleFile
):
1978 # if we have --repeat, duplicate the tests as needed
1979 for i
in range(0, options
.get("repeat", 0) + 1):
1984 verbose
=self
.verbose
or test_object
.get("verbose") == "true",
1985 usingTSan
=usingTSan
,
1986 usingCrashReporter
=usingCrashReporter
,
1987 mobileArgs
=mobileArgs
,
1990 if "run-sequentially" in test_object
or self
.sequential
:
1991 sequential_tests
.append(test
)
1993 tests_queue
.append(test
)
1995 status
= self
.runTestList(
1996 tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2000 # Test verification: Run each test many times, in various configurations,
2001 # in hopes of finding intermittent failures.
2005 # Run tests sequentially. Parallel mode would also work, except that
2006 # the logging system gets confused when 2 or more tests with the same
2007 # name run at the same time.
2008 sequential_tests
= []
2009 for i
in range(VERIFY_REPEAT
):
2012 test_object
, retry
=False, mobileArgs
=mobileArgs
, **kwargs
2014 sequential_tests
.append(test
)
2015 status
= self
.runTestList(
2016 tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2021 # Run tests sequentially, with MOZ_CHAOSMODE enabled.
2022 sequential_tests
= []
2023 self
.env
["MOZ_CHAOSMODE"] = "0xfb"
2024 # chaosmode runs really slow, allow tests extra time to pass
2025 self
.harness_timeout
= self
.harness_timeout
* 2
2026 for i
in range(VERIFY_REPEAT
):
2029 test_object
, retry
=False, mobileArgs
=mobileArgs
, **kwargs
2031 sequential_tests
.append(test
)
2032 status
= self
.runTestList(
2033 tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2035 self
.harness_timeout
= self
.harness_timeout
/ 2
2039 ("1. Run each test %d times, sequentially." % VERIFY_REPEAT
, step1
),
2041 "2. Run each test %d times, sequentially, in chaos mode."
2046 startTime
= datetime
.now()
2047 maxTime
= timedelta(seconds
=options
["verifyMaxTime"])
2048 for test_object
in self
.alltests
:
2050 for descr
, step
in steps
:
2051 stepResults
[descr
] = "not run / incomplete"
2052 finalResult
= "PASSED"
2053 for descr
, step
in steps
:
2054 if (datetime
.now() - startTime
) > maxTime
:
2056 "::: Test verification is taking too long: Giving up!"
2059 "::: So far, all checks passed, but not "
2060 "all checks were run."
2063 self
.log
.info(":::")
2064 self
.log
.info('::: Running test verification step "%s"...' % descr
)
2065 self
.log
.info(":::")
2067 if status
is not True:
2068 stepResults
[descr
] = "FAIL"
2069 finalResult
= "FAILED!"
2071 stepResults
[descr
] = "Pass"
2072 self
.log
.info(":::")
2074 "::: Test verification summary for: %s" % test_object
["path"]
2076 self
.log
.info(":::")
2077 for descr
in sorted(stepResults
.keys()):
2078 self
.log
.info("::: %s : %s" % (descr
, stepResults
[descr
]))
2079 self
.log
.info(":::")
2080 self
.log
.info("::: Test verification %s" % finalResult
)
2081 self
.log
.info(":::")
2084 self
.shutdownHttp3Server()
2088 def start_test(self
, test
):
2091 def test_ended(self
, test
):
2095 self
, tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2098 self
.log
.info("Running tests sequentially.")
2100 self
.log
.info("Using at most %d threads." % self
.threadCount
)
2102 # keep a set of threadCount running tests and start running the
2103 # tests in the queue at most threadCount at a time
2104 running_tests
= set()
2109 self
.try_again_list
= []
2111 tests_by_manifest
= defaultdict(list)
2112 for test
in self
.alltests
:
2113 group
= get_full_group_name(test
)
2114 tests_by_manifest
[group
].append(test
["id"])
2116 self
.log
.suite_start(tests_by_manifest
, name
="xpcshell")
2118 while tests_queue
or running_tests
:
2119 # if we're not supposed to continue and all of the running tests
2121 if not keep_going
and not running_tests
:
2124 # if there's room to run more tests, start running them
2126 keep_going
and tests_queue
and (len(running_tests
) < self
.threadCount
)
2128 test
= tests_queue
.popleft()
2129 running_tests
.add(test
)
2130 self
.start_test(test
)
2132 # queue is full (for now) or no more new tests,
2133 # process the finished tests so far
2135 # wait for at least one of the tests to finish
2139 # find what tests are done (might be more than 1)
2141 for test
in running_tests
:
2143 self
.test_ended(test
)
2144 done_tests
.add(test
)
2147 ) # join with timeout so we don't hang on blocked threads
2148 # if the test had trouble, we will try running it again
2149 # at the end of the run
2150 if test
.retry
or test
.is_alive():
2151 # if the join call timed out, test.is_alive => True
2152 self
.try_again_list
.append(test
.test_object
)
2154 # did the test encounter any exception?
2156 exceptions
.append(test
.exception
)
2157 tracebacks
.append(test
.traceback
)
2158 # we won't add any more tests, will just wait for
2159 # the currently running ones to finish
2161 infra_abort
= infra_abort
and test
.infra
2162 keep_going
= keep_going
and test
.keep_going
2163 self
.addTestResults(test
)
2165 # make room for new tests to run
2166 running_tests
.difference_update(done_tests
)
2169 return TBPL_RETRY
# terminate early
2172 # run the other tests sequentially
2173 for test
in sequential_tests
:
2176 "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so "
2177 "stopped run. (Use --keep-going to keep running tests "
2178 "after killing one with SIGINT)"
2181 self
.start_test(test
)
2183 self
.test_ended(test
)
2184 if (test
.failCount
> 0 or test
.passCount
<= 0) and os
.environ
.get(
2187 self
.try_again_list
.append(test
.test_object
)
2189 self
.addTestResults(test
)
2190 # did the test encounter any exception?
2192 exceptions
.append(test
.exception
)
2193 tracebacks
.append(test
.traceback
)
2195 keep_going
= test
.keep_going
2197 # retry tests that failed when run in parallel
2198 if self
.try_again_list
:
2199 self
.log
.info("Retrying tests that failed when run in parallel.")
2200 for test_object
in self
.try_again_list
:
2204 verbose
=self
.verbose
,
2205 mobileArgs
=mobileArgs
,
2208 self
.start_test(test
)
2210 self
.test_ended(test
)
2211 self
.addTestResults(test
)
2212 # did the test encounter any exception?
2214 exceptions
.append(test
.exception
)
2215 tracebacks
.append(test
.traceback
)
2217 keep_going
= test
.keep_going
2219 # restore default SIGINT behaviour
2220 signal
.signal(signal
.SIGINT
, signal
.SIG_DFL
)
2222 # Clean up any slacker directories that might be lying around
2223 # Some might fail because of windows taking too long to unlock them.
2224 # We don't do anything if this fails because the test machines will have
2225 # their $TEMP dirs cleaned up on reboot anyway.
2226 for directory
in self
.cleanup_dir_list
:
2228 shutil
.rmtree(directory
)
2230 self
.log
.info("%s could not be cleaned up." % directory
)
2233 self
.log
.info("Following exceptions were raised:")
2234 for t
in tracebacks
:
2238 if self
.testCount
== 0 and os
.environ
.get("MOZ_AUTOMATION") != "1":
2239 self
.log
.error("No tests run. Did you pass an invalid --test-path?")
2242 # doing this allows us to pass the mozharness parsers that
2243 # report an orange job for failCount>0
2244 if self
.runFailures
:
2245 passed
= self
.passCount
2246 self
.passCount
= self
.failCount
2247 self
.failCount
= passed
2249 self
.log
.info("INFO | Result summary:")
2250 self
.log
.info("INFO | Passed: %d" % self
.passCount
)
2251 self
.log
.info("INFO | Failed: %d" % self
.failCount
)
2252 self
.log
.info("INFO | Todo: %d" % self
.todoCount
)
2253 self
.log
.info("INFO | Retried: %d" % len(self
.try_again_list
))
2255 if gotSIGINT
and not keep_going
:
2257 "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. "
2258 "(Use --keep-going to keep running tests after "
2259 "killing one with SIGINT)"
2263 self
.log
.suite_end()
2264 return self
.runFailures
or self
.failCount
== 0
2268 parser
= parser_desktop()
2269 options
= parser
.parse_args()
2271 log
= commandline
.setup_logging("XPCShell", options
, {"tbpl": sys
.stdout
})
2273 if options
.xpcshell
is None and options
.app_binary
is None:
2275 "Must provide path to xpcshell using --xpcshell or Firefox using --app-binary"
2279 if options
.xpcshell
is not None and options
.app_binary
is not None:
2281 "Cannot provide --xpcshell and --app-binary - they are mutually exclusive options. Choose one."
2285 xpcsh
= XPCShellTests(log
)
2287 if options
.interactive
and not options
.testPath
:
2288 log
.error("Error: You must specify a test filename in interactive mode!")
2291 result
= xpcsh
.runTests(options
)
2292 if result
== TBPL_RETRY
:
2299 if __name__
== "__main__":