3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
21 from argparse
import Namespace
22 from collections
import defaultdict
, deque
, namedtuple
23 from contextlib
import contextmanager
24 from datetime
import datetime
, timedelta
25 from functools
import partial
26 from multiprocessing
import cpu_count
27 from subprocess
import PIPE
, STDOUT
, Popen
28 from tempfile
import gettempdir
, mkdtemp
29 from threading
import Event
, Thread
, Timer
, current_thread
33 from mozserve
import Http3Server
42 from xpcshellcommandline
import parser_desktop
44 SCRIPT_DIR
= os
.path
.abspath(os
.path
.realpath(os
.path
.dirname(__file__
)))
47 from mozbuild
.base
import MozbuildObject
49 build
= MozbuildObject
.from_environment(cwd
=SCRIPT_DIR
)
53 HARNESS_TIMEOUT
= 5 * 60
54 TBPL_RETRY
= 4 # defined in mozharness
56 # benchmarking on tbpl revealed that this works best for now
57 # TODO: This has been evaluated/set many years ago and we might want to
58 # benchmark this again.
59 # These days with e10s/fission the number of real processes/threads running
60 # can be significantly higher, with both consequences on runtime and memory
61 # consumption. So be aware that NUM_THREADS is just saying how many tests will
62 # be started maximum in parallel and that depending on the tests there is
63 # only a weak correlation to the effective number of processes or threads.
64 # Be also aware that we can override this value with the threadCount option
65 # on the command line to tweak it for a concrete CPU/memory combination.
66 NUM_THREADS
= int(cpu_count() * 4)
67 if sys
.platform
== "win32":
68 NUM_THREADS
= NUM_THREADS
/ 2
70 EXPECTED_LOG_ACTIONS
= set(
72 "crash_reporter_init",
78 # --------------------------------------------------------------
79 # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
81 here
= os
.path
.dirname(__file__
)
82 mozbase
= os
.path
.realpath(os
.path
.join(os
.path
.dirname(here
), "mozbase"))
84 if os
.path
.isdir(mozbase
):
85 for package
in os
.listdir(mozbase
):
86 sys
.path
.append(os
.path
.join(mozbase
, package
))
91 from manifestparser
import TestManifest
92 from manifestparser
.filters
import chunk_by_slice
, failures
, pathprefix
, tags
93 from manifestparser
.util
import normsep
94 from mozlog
import commandline
95 from mozprofile
import Profile
96 from mozprofile
.cli
import parse_preferences
97 from mozrunner
.utils
import get_stack_fixer_function
99 # --------------------------------------------------------------
101 # TODO: perhaps this should be in a more generally shared location?
102 # This regex matches all of the C0 and C1 control characters
103 # (U+0000 through U+001F; U+007F; U+0080 through U+009F),
104 # except TAB (U+0009), CR (U+000D), LF (U+000A) and backslash (U+005C).
105 # A raw string is deliberately not used.
106 _cleanup_encoding_re
= re
.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\\\\]")
109 def get_full_group_name(test
):
110 group
= test
["manifest"]
111 if "ancestor_manifest" in test
:
112 ancestor_manifest
= normsep(test
["ancestor_manifest"])
113 # Only change the group id if ancestor is not the generated root manifest.
114 if "/" in ancestor_manifest
:
115 group
= "{}:{}".format(ancestor_manifest
, group
)
119 def _cleanup_encoding_repl(m
):
121 return "\\\\" if c
== "\\" else "\\x{0:02X}".format(ord(c
))
124 def cleanup_encoding(s
):
125 """S is either a byte or unicode string. Either way it may
126 contain control characters, unpaired surrogates, reserved code
127 points, etc. If it is a byte string, it is assumed to be
128 UTF-8, but it may not be *correct* UTF-8. Return a
129 sanitized unicode object."""
130 if not isinstance(s
, six
.string_types
):
131 if isinstance(s
, six
.binary_type
):
132 return six
.ensure_str(s
)
134 return six
.text_type(s
)
135 if isinstance(s
, six
.binary_type
):
136 s
= s
.decode("utf-8", "replace")
137 # Replace all C0 and C1 control characters with \xNN escapes.
138 return _cleanup_encoding_re
.sub(_cleanup_encoding_repl
, s
)
142 def popenCleanupHack():
144 Hack to work around https://bugs.python.org/issue37380
145 The basic idea is that on old versions of Python on Windows,
146 we need to clear subprocess._cleanup before we call Popen(),
147 then restore it afterwards.
150 if mozinfo
.isWin
and sys
.version_info
[0] == 3 and sys
.version_info
< (3, 7, 5):
151 savedCleanup
= subprocess
._cleanup
152 subprocess
._cleanup
= lambda: None
157 subprocess
._cleanup
= savedCleanup
160 """ Control-C handling """
164 def markGotSIGINT(signum
, stackFrame
):
169 class XPCShellTestThread(Thread
):
176 usingCrashReporter
=False,
179 Thread
.__init
__(self
)
182 self
.test_object
= test_object
184 self
.verbose
= verbose
185 self
.usingTSan
= usingTSan
186 self
.usingCrashReporter
= usingCrashReporter
188 self
.appPath
= kwargs
.get("appPath")
189 self
.xrePath
= kwargs
.get("xrePath")
190 self
.utility_path
= kwargs
.get("utility_path")
191 self
.testingModulesDir
= kwargs
.get("testingModulesDir")
192 self
.debuggerInfo
= kwargs
.get("debuggerInfo")
193 self
.jsDebuggerInfo
= kwargs
.get("jsDebuggerInfo")
194 self
.headJSPath
= kwargs
.get("headJSPath")
195 self
.testharnessdir
= kwargs
.get("testharnessdir")
196 self
.profileName
= kwargs
.get("profileName")
197 self
.singleFile
= kwargs
.get("singleFile")
198 self
.env
= copy
.deepcopy(kwargs
.get("env"))
199 self
.symbolsPath
= kwargs
.get("symbolsPath")
200 self
.logfiles
= kwargs
.get("logfiles")
201 self
.app_binary
= kwargs
.get("app_binary")
202 self
.xpcshell
= kwargs
.get("xpcshell")
203 self
.xpcsRunArgs
= kwargs
.get("xpcsRunArgs")
204 self
.failureManifest
= kwargs
.get("failureManifest")
205 self
.jscovdir
= kwargs
.get("jscovdir")
206 self
.stack_fixer_function
= kwargs
.get("stack_fixer_function")
207 self
._rootTempDir
= kwargs
.get("tempDir")
208 self
.cleanup_dir_list
= kwargs
.get("cleanup_dir_list")
209 self
.pStdout
= kwargs
.get("pStdout")
210 self
.pStderr
= kwargs
.get("pStderr")
211 self
.keep_going
= kwargs
.get("keep_going")
212 self
.log
= kwargs
.get("log")
213 self
.app_dir_key
= kwargs
.get("app_dir_key")
214 self
.interactive
= kwargs
.get("interactive")
215 self
.rootPrefsFile
= kwargs
.get("rootPrefsFile")
216 self
.extraPrefs
= kwargs
.get("extraPrefs")
217 self
.verboseIfFails
= kwargs
.get("verboseIfFails")
218 self
.headless
= kwargs
.get("headless")
219 self
.runFailures
= kwargs
.get("runFailures")
220 self
.timeoutAsPass
= kwargs
.get("timeoutAsPass")
221 self
.crashAsPass
= kwargs
.get("crashAsPass")
222 self
.conditionedProfileDir
= kwargs
.get("conditionedProfileDir")
226 # Default the test prefsFile to the rootPrefsFile.
227 self
.prefsFile
= self
.rootPrefsFile
229 # only one of these will be set to 1. adding them to the totals in
235 # Context for output processing
236 self
.output_lines
= []
237 self
.has_failure_output
= False
238 self
.saw_crash_reporter_init
= False
239 self
.saw_proc_start
= False
240 self
.saw_proc_end
= False
242 self
.harness_timeout
= kwargs
.get("harness_timeout")
243 self
.timedout
= False
246 # event from main thread to signal work done
247 self
.event
= kwargs
.get("event")
248 self
.done
= False # explicitly set flag so we don't rely on thread.isAlive
253 except PermissionError
as e
:
256 self
.traceback
= traceback
.format_exc()
257 except Exception as e
:
259 self
.traceback
= traceback
.format_exc()
261 self
.exception
= None
262 self
.traceback
= None
265 "%s failed or timed out, will retry." % self
.test_object
["id"]
270 def kill(self
, proc
):
272 Simple wrapper to kill a process.
273 On a remote system, this is overloaded to handle remote process communication.
277 def removeDir(self
, dirname
):
279 Simple wrapper to remove (recursively) a given directory.
280 On a remote system, we need to overload this to work on the remote filesystem.
282 mozfile
.remove(dirname
)
284 def poll(self
, proc
):
286 Simple wrapper to check if a process has terminated.
287 On a remote system, this is overloaded to handle remote process communication.
291 def createLogFile(self
, test_file
, stdout
):
293 For a given test file and stdout buffer, create a log file.
294 On a remote system we have to fix the test name since it can contain directories.
296 with
open(test_file
+ ".log", "w") as f
:
299 def getReturnCode(self
, proc
):
301 Simple wrapper to get the return code for a given process.
302 On a remote system we overload this to work with the remote process management.
304 if proc
is not None and hasattr(proc
, "returncode"):
305 return proc
.returncode
308 def communicate(self
, proc
):
310 Simple wrapper to communicate with a process.
311 On a remote system, this is overloaded to handle remote process communication.
313 # Processing of incremental output put here to
314 # sidestep issues on remote platforms, where what we know
315 # as proc is a file pulled off of a device.
318 line
= proc
.stdout
.readline()
321 self
.process_line(line
)
323 if self
.saw_proc_start
and not self
.saw_proc_end
:
324 self
.has_failure_output
= True
326 return proc
.communicate()
329 self
, cmd
, stdout
, stderr
, env
, cwd
, timeout
=None, test_name
=None
332 Simple wrapper to launch a process.
333 On a remote system, this is more complex and we need to overload this function.
335 # timeout is needed by remote xpcshell to extend the
336 # remote device timeout. It is not used in this function.
338 cwd
= six
.ensure_str(cwd
)
339 for i
in range(len(cmd
)):
340 cmd
[i
] = six
.ensure_str(cmd
[i
])
343 popen_func
= psutil
.Popen
347 with
popenCleanupHack():
348 proc
= popen_func(cmd
, stdout
=stdout
, stderr
=stderr
, env
=env
, cwd
=cwd
)
352 def checkForCrashes(self
, dump_directory
, symbols_path
, test_name
=None):
354 Simple wrapper to check for crashes.
355 On a remote system, this is more complex and we need to overload this function.
361 return mozcrash
.log_crashes(
362 self
.log
, dump_directory
, symbols_path
, test
=test_name
, quiet
=quiet
365 def logCommand(self
, name
, completeCmd
, testdir
):
366 self
.log
.info("%s | full command: %r" % (name
, completeCmd
))
367 self
.log
.info("%s | current directory: %r" % (name
, testdir
))
368 # Show only those environment variables that are changed from
369 # the ambient environment.
370 changedEnv
= set("%s=%s" % i
for i
in six
.iteritems(self
.env
)) - set(
371 "%s=%s" % i
for i
in six
.iteritems(os
.environ
)
373 self
.log
.info("%s | environment: %s" % (name
, list(changedEnv
)))
374 shell_command_tokens
= [
375 pipes
.quote(tok
) for tok
in list(changedEnv
) + completeCmd
378 "%s | as shell command: (cd %s; %s)"
379 % (name
, pipes
.quote(testdir
), " ".join(shell_command_tokens
))
382 def killTimeout(self
, proc
):
383 if proc
is not None and hasattr(proc
, "pid"):
384 mozcrash
.kill_and_get_minidump(
385 proc
.pid
, self
.tempDir
, utility_path
=self
.utility_path
388 self
.log
.info("not killing -- proc or pid unknown")
390 def postCheck(self
, proc
):
391 """Checks for a still-running test process, kills it and fails the test if found.
392 We can sometimes get here before the process has terminated, which would
393 cause removeDir() to fail - so check for the process and kill it if needed.
395 if proc
and self
.poll(proc
) is None:
399 except psutil
.NoSuchProcess
:
403 message
= "%s | Process still running after test!" % self
.test_object
["id"]
405 self
.log
.info(message
)
408 self
.log
.error(message
)
409 self
.log_full_output()
412 def testTimeout(self
, proc
):
413 if self
.test_object
["expected"] == "pass":
420 self
.test_object
["id"],
423 message
="Test timed out",
427 if self
.timeoutAsPass
:
432 self
.test_object
["id"],
435 message
="Test timed out",
437 self
.log_full_output()
441 self
.killTimeout(proc
)
442 self
.log
.info("xpcshell return code: %s" % self
.getReturnCode(proc
))
444 self
.clean_temp_dirs(self
.test_object
["path"])
446 def updateTestPrefsFile(self
):
447 # If the Manifest file has some additional prefs, merge the
448 # prefs set in the user.js file stored in the _rootTempdir
449 # with the prefs from the manifest and the prefs specified
450 # in the extraPrefs option.
451 if "prefs" in self
.test_object
:
452 # Merge the user preferences in a fake profile dir in a
453 # local temporary dir (self.tempDir is the remoteTmpDir
454 # for the RemoteXPCShellTestThread subclass and so we
455 # can't use that tempDir here).
456 localTempDir
= mkdtemp(prefix
="xpc-other-", dir=self
._rootTempDir
)
459 interpolation
= {"server": "dummyserver"}
460 profile
= Profile(profile
=localTempDir
, restore
=False)
461 # _rootTempDir contains a user.js file, generated by buildPrefsFile
462 profile
.merge(self
._rootTempDir
, interpolation
=interpolation
)
464 prefs
= self
.test_object
["prefs"].strip().split()
465 name
= self
.test_object
["id"]
468 "%s: Per-test extra prefs will be set:\n {}".format(
474 profile
.set_preferences(parse_preferences(prefs
), filename
=filename
)
475 # Make sure that the extra prefs form the command line are overriding
476 # any prefs inherited from the shared profile data or the manifest prefs.
477 profile
.set_preferences(
478 parse_preferences(self
.extraPrefs
), filename
=filename
480 return os
.path
.join(profile
.profile
, filename
)
482 # Return the root prefsFile if there is no other prefs to merge.
483 # This is the path set by buildPrefsFile.
484 return self
.rootPrefsFile
487 def conditioned_profile_copy(self
):
488 """Returns a copy of the original conditioned profile that was created."""
490 condprof_copy
= os
.path
.join(tempfile
.mkdtemp(), "profile")
492 self
.conditionedProfileDir
,
494 ignore
=shutil
.ignore_patterns("lock"),
496 self
.log
.info("Created a conditioned-profile copy: %s" % condprof_copy
)
499 def buildCmdTestFile(self
, name
):
501 Build the command line arguments for the test file.
502 On a remote system, this may be overloaded to use a remote path structure.
504 return ["-e", 'const _TEST_FILE = ["%s"];' % name
.replace("\\", "/")]
506 def setupTempDir(self
):
507 tempDir
= mkdtemp(prefix
="xpc-other-", dir=self
._rootTempDir
)
508 self
.env
["XPCSHELL_TEST_TEMP_DIR"] = tempDir
510 self
.log
.info("temp dir is %s" % tempDir
)
513 def setupProfileDir(self
):
515 Create a temporary folder for the profile and set appropriate environment variables.
516 When running check-interactive and check-one, the directory is well-defined and
517 retained for inspection once the tests complete.
519 On a remote system, this may be overloaded to use a remote path structure.
521 if self
.conditionedProfileDir
:
522 profileDir
= self
.conditioned_profile_copy
523 elif self
.interactive
or self
.singleFile
:
524 profileDir
= os
.path
.join(gettempdir(), self
.profileName
, "xpcshellprofile")
526 # This could be left over from previous runs
527 self
.removeDir(profileDir
)
530 os
.makedirs(profileDir
)
532 profileDir
= mkdtemp(prefix
="xpc-profile-", dir=self
._rootTempDir
)
533 self
.env
["XPCSHELL_TEST_PROFILE_DIR"] = profileDir
534 if self
.interactive
or self
.singleFile
:
535 self
.log
.info("profile dir is %s" % profileDir
)
538 def setupMozinfoJS(self
):
539 mozInfoJSPath
= os
.path
.join(self
.profileDir
, "mozinfo.json")
540 mozInfoJSPath
= mozInfoJSPath
.replace("\\", "\\\\")
541 mozinfo
.output_to_file(mozInfoJSPath
)
544 def buildCmdHead(self
):
546 Build the command line arguments for the head files,
547 along with the address of the webserver which some tests require.
549 On a remote system, this is overloaded to resolve quoting issues over a
550 secondary command line.
552 headfiles
= self
.getHeadFiles(self
.test_object
)
553 cmdH
= ", ".join(['"' + f
.replace("\\", "/") + '"' for f
in headfiles
])
555 dbgport
= 0 if self
.jsDebuggerInfo
is None else self
.jsDebuggerInfo
.port
559 "const _HEAD_FILES = [%s];" % cmdH
,
561 "const _JSDEBUGGER_PORT = %d;" % dbgport
,
564 def getHeadFiles(self
, test
):
565 """Obtain lists of head- files. Returns a list of head files."""
567 def sanitize_list(s
, kind
):
568 for f
in s
.strip().split(" "):
573 path
= os
.path
.normpath(os
.path
.join(test
["here"], f
))
574 if not os
.path
.exists(path
):
575 raise Exception("%s file does not exist: %s" % (kind
, path
))
577 if not os
.path
.isfile(path
):
578 raise Exception("%s file is not a file: %s" % (kind
, path
))
582 headlist
= test
.get("head", "")
583 return list(sanitize_list(headlist
, "head"))
585 def buildXpcsCmd(self
):
587 Load the root head.js file as the first file in our test path, before other head,
588 and test files. On a remote system, we overload this to add additional command
589 line arguments, so this gets overloaded.
591 # - NOTE: if you rename/add any of the constants set here, update
592 # do_load_child_test_harness() in head.js
594 self
.appPath
= self
.xrePath
613 'const _HEAD_JS_PATH = "%s";' % self
.headJSPath
,
615 'const _MOZINFO_JS_PATH = "%s";' % self
.mozInfoJSPath
,
617 'const _PREFS_FILE = "%s";' % self
.prefsFile
.replace("\\", "\\\\"),
620 if self
.testingModulesDir
:
621 # Escape backslashes in string literal.
622 sanitized
= self
.testingModulesDir
.replace("\\", "\\\\")
623 xpcsCmd
.extend(["-e", 'const _TESTING_MODULES_DIR = "%s";' % sanitized
])
625 xpcsCmd
.extend(["-f", os
.path
.join(self
.testharnessdir
, "head.js")])
627 if self
.debuggerInfo
:
628 xpcsCmd
= [self
.debuggerInfo
.path
] + self
.debuggerInfo
.args
+ xpcsCmd
632 def cleanupDir(self
, directory
, name
):
633 if not os
.path
.exists(directory
):
636 # up to TRY_LIMIT attempts (one every second), because
637 # the Windows filesystem is slow to react to the changes
640 while try_count
< TRY_LIMIT
:
642 self
.removeDir(directory
)
644 self
.log
.info("Failed to remove directory: %s. Waiting." % directory
)
645 # We suspect the filesystem may still be making changes. Wait a
646 # little bit and try again.
653 # we try cleaning up again later at the end of the run
654 self
.cleanup_dir_list
.append(directory
)
656 def clean_temp_dirs(self
, name
):
657 # We don't want to delete the profile when running check-interactive
659 if self
.profileDir
and not self
.interactive
and not self
.singleFile
:
660 self
.cleanupDir(self
.profileDir
, name
)
662 self
.cleanupDir(self
.tempDir
, name
)
664 def parse_output(self
, output
):
665 """Parses process output for structured messages and saves output as it is
666 read. Sets self.has_failure_output in case of evidence of a failure"""
667 for line_string
in output
.splitlines():
668 self
.process_line(line_string
)
670 if self
.saw_proc_start
and not self
.saw_proc_end
:
671 self
.has_failure_output
= True
673 def fix_text_output(self
, line
):
674 line
= cleanup_encoding(line
)
675 if self
.stack_fixer_function
is not None:
676 line
= self
.stack_fixer_function(line
)
678 if isinstance(line
, bytes
):
679 line
= line
.decode("utf-8")
682 def log_line(self
, line
):
683 """Log a line of output (either a parser json object or text output from
685 if isinstance(line
, six
.string_types
) or isinstance(line
, bytes
):
686 line
= self
.fix_text_output(line
).rstrip("\r\n")
687 self
.log
.process_output(self
.proc_ident
, line
, command
=self
.command
)
689 if "message" in line
:
690 line
["message"] = self
.fix_text_output(line
["message"])
691 if "xpcshell_process" in line
:
692 line
["thread"] = " ".join(
693 [current_thread().name
, line
["xpcshell_process"]]
696 line
["thread"] = current_thread().name
697 self
.log
.log_raw(line
)
699 def log_full_output(self
):
700 """Logs any buffered output from the test process, and clears the buffer."""
701 if not self
.output_lines
:
703 self
.log
.info(">>>>>>>")
704 for line
in self
.output_lines
:
706 self
.log
.info("<<<<<<<")
707 self
.output_lines
= []
709 def report_message(self
, message
):
710 """Stores or logs a json log message in mozlog format."""
712 self
.log_line(message
)
714 self
.output_lines
.append(message
)
716 def process_line(self
, line_string
):
717 """Parses a single line of output, determining its significance and
720 if isinstance(line_string
, bytes
):
721 # Transform binary to string representation
722 line_string
= line_string
.decode(sys
.stdout
.encoding
, errors
="replace")
724 if not line_string
.strip():
728 line_object
= json
.loads(line_string
)
729 if not isinstance(line_object
, dict):
730 self
.report_message(line_string
)
733 self
.report_message(line_string
)
737 "action" not in line_object
738 or line_object
["action"] not in EXPECTED_LOG_ACTIONS
740 # The test process output JSON.
741 self
.report_message(line_string
)
744 if line_object
["action"] == "crash_reporter_init":
745 self
.saw_crash_reporter_init
= True
748 action
= line_object
["action"]
750 self
.has_failure_output
= (
751 self
.has_failure_output
752 or "expected" in line_object
754 and line_object
["level"] == "ERROR"
757 self
.report_message(line_object
)
759 if action
== "log" and line_object
["message"] == "CHILD-TEST-STARTED":
760 self
.saw_proc_start
= True
761 elif action
== "log" and line_object
["message"] == "CHILD-TEST-COMPLETED":
762 self
.saw_proc_end
= True
765 """Run an individual xpcshell test."""
768 name
= self
.test_object
["id"]
769 path
= self
.test_object
["path"]
770 group
= get_full_group_name(self
.test_object
)
772 # Check for skipped tests
773 if "disabled" in self
.test_object
:
774 message
= self
.test_object
["disabled"]
776 message
= "disabled from xpcshell manifest"
777 self
.log
.test_start(name
, group
=group
)
778 self
.log
.test_end(name
, "SKIP", message
=message
, group
=group
)
781 self
.keep_going
= True
784 # Check for known-fail tests
785 expect_pass
= self
.test_object
["expected"] == "pass"
787 if self
.test_object
.get("disable_e10s") == "true":
788 self
.env
["MOZ_FORCE_DISABLE_E10S"] = "1"
790 # By default self.appPath will equal the gre dir. If specified in the
791 # xpcshell.toml file, set a different app dir for this test.
792 if self
.app_dir_key
and self
.app_dir_key
in self
.test_object
:
793 rel_app_dir
= self
.test_object
[self
.app_dir_key
]
794 rel_app_dir
= os
.path
.join(self
.xrePath
, rel_app_dir
)
795 self
.appPath
= os
.path
.abspath(rel_app_dir
)
799 test_dir
= os
.path
.dirname(path
)
801 # Create a profile and a temp dir that the JS harness can stick
802 # a profile and temporary data in
803 self
.profileDir
= self
.setupProfileDir()
804 self
.tempDir
= self
.setupTempDir()
805 self
.mozInfoJSPath
= self
.setupMozinfoJS()
807 # Setup per-manifest prefs and write them into the tempdir.
808 self
.prefsFile
= self
.updateTestPrefsFile()
810 # The order of the command line is important:
811 # 1) Arguments for xpcshell itself
812 self
.command
= self
.buildXpcsCmd()
814 # 2) Arguments for the head files
815 self
.command
.extend(self
.buildCmdHead())
817 # 3) Arguments for the test file
818 self
.command
.extend(self
.buildCmdTestFile(path
))
819 self
.command
.extend(["-e", 'const _TEST_NAME = "%s";' % name
])
821 # 4) Arguments for code coverage
824 ["-e", 'const _JSCOV_DIR = "%s";' % self
.jscovdir
.replace("\\", "/")]
827 # 5) Runtime arguments
828 if "debug" in self
.test_object
:
829 self
.command
.append("-d")
831 self
.command
.extend(self
.xpcsRunArgs
)
833 if self
.test_object
.get("dmd") == "true":
834 self
.env
["PYTHON"] = sys
.executable
835 self
.env
["BREAKPAD_SYMBOLS_PATH"] = self
.symbolsPath
837 if self
.test_object
.get("snap") == "true":
838 self
.env
["SNAP_NAME"] = "firefox"
839 self
.env
["SNAP_INSTANCE_NAME"] = "firefox"
841 if self
.test_object
.get("subprocess") == "true":
842 self
.env
["PYTHON"] = sys
.executable
845 self
.test_object
.get("headless", "true" if self
.headless
else None)
848 self
.env
["MOZ_HEADLESS"] = "1"
849 self
.env
["DISPLAY"] = "77" # Set a fake display.
851 testTimeoutInterval
= self
.harness_timeout
852 # Allow a test to request a multiple of the timeout if it is expected to take long
853 if "requesttimeoutfactor" in self
.test_object
:
854 testTimeoutInterval
*= int(self
.test_object
["requesttimeoutfactor"])
857 if not self
.interactive
and not self
.debuggerInfo
and not self
.jsDebuggerInfo
:
858 testTimer
= Timer(testTimeoutInterval
, lambda: self
.testTimeout(proc
))
862 process_output
= None
865 self
.log
.test_start(name
, group
=group
)
867 self
.logCommand(name
, self
.command
, test_dir
)
869 proc
= self
.launchProcess(
875 timeout
=testTimeoutInterval
,
879 if hasattr(proc
, "pid"):
880 self
.proc_ident
= proc
.pid
882 # On mobile, "proc" is just a file.
883 self
.proc_ident
= name
886 self
.log
.info("%s | Process ID: %d" % (name
, self
.proc_ident
))
888 # Communicate returns a tuple of (stdout, stderr), however we always
889 # redirect stderr to stdout, so the second element is ignored.
890 process_output
, _
= self
.communicate(proc
)
893 # Not sure what else to do here...
894 self
.keep_going
= True
901 # For the remote case, stdout is not yet depleted, so we parse
902 # it here all at once.
903 self
.parse_output(process_output
)
905 return_code
= self
.getReturnCode(proc
)
907 # TSan'd processes return 66 if races are detected. This isn't
908 # good in the sense that there's no way to distinguish between
909 # a process that would normally have returned zero but has races,
910 # and a race-free process that returns 66. But I don't see how
911 # to do better. This ambiguity is at least constrained to the
912 # with-TSan case. It doesn't affect normal builds.
914 # This also assumes that the magic value 66 isn't overridden by
915 # a TSAN_OPTIONS=exitcode=<number> environment variable setting.
917 TSAN_EXIT_CODE_WITH_RACES
= 66
919 return_code_ok
= return_code
== 0 or (
920 self
.usingTSan
and return_code
== TSAN_EXIT_CODE_WITH_RACES
923 # Due to the limitation on the remote xpcshell test, the process
924 # return code does not represent the process crash.
925 # If crash_reporter_init log has not been seen and the return code
926 # is 0, it means the process crashed before setting up the crash
929 # NOTE: Crash reporter is not enabled on some configuration, such
930 # as ASAN and TSAN. Those configuration shouldn't be using
931 # remote xpcshell test, and the crash should be caught by
932 # the process return code.
933 # NOTE: self.saw_crash_reporter_init is False also when adb failed
934 # to launch process, and in that case the return code is
936 # (see launchProcess in remotexpcshelltests.py)
937 ended_before_crash_reporter_init
= (
939 and self
.usingCrashReporter
940 and not self
.saw_crash_reporter_init
944 (not self
.has_failure_output
)
945 and not ended_before_crash_reporter_init
949 status
= "PASS" if passed
else "FAIL"
950 expected
= "PASS" if expect_pass
else "FAIL"
951 message
= "xpcshell return code: %d" % return_code
956 if status
!= expected
or ended_before_crash_reporter_init
:
957 if ended_before_crash_reporter_init
:
962 message
="Test ended before setting up the crash reporter",
970 message
="Test failed or timed out, will retry",
973 self
.clean_temp_dirs(path
)
974 if self
.verboseIfFails
and not self
.verbose
:
975 self
.log_full_output()
979 name
, status
, expected
=expected
, message
=message
, group
=group
981 self
.log_full_output()
985 if self
.failureManifest
:
986 with
open(self
.failureManifest
, "a") as f
:
987 f
.write("[%s]\n" % self
.test_object
["path"])
988 for k
, v
in self
.test_object
.items():
989 f
.write("%s = %s\n" % (k
, v
))
992 # If TSan reports a race, dump the output, else we can't
993 # diagnose what the problem was. See comments above about
994 # the significance of TSAN_EXIT_CODE_WITH_RACES.
995 if self
.usingTSan
and return_code
== TSAN_EXIT_CODE_WITH_RACES
:
996 self
.log_full_output()
999 name
, status
, expected
=expected
, message
=message
, group
=group
1002 self
.log_full_output()
1011 if self
.checkForCrashes(self
.tempDir
, self
.symbolsPath
, test_name
=name
):
1013 self
.clean_temp_dirs(path
)
1016 # If we assert during shutdown there's a chance the test has passed
1017 # but we haven't logged full output, so do so here.
1018 self
.log_full_output()
1021 if self
.logfiles
and process_output
:
1022 self
.createLogFile(name
, process_output
)
1025 self
.postCheck(proc
)
1026 self
.clean_temp_dirs(path
)
1029 self
.log
.error("Received SIGINT (control-C) during test execution")
1033 self
.keep_going
= False
1036 self
.keep_going
= True
1039 class XPCShellTests(object):
1040 def __init__(self
, log
=None):
1041 """Initializes node status and logger."""
1043 self
.harness_timeout
= HARNESS_TIMEOUT
1045 self
.http3Server
= None
1046 self
.conditioned_profile_dir
= None
1048 def getTestManifest(self
, manifest
):
1049 if isinstance(manifest
, TestManifest
):
1051 elif manifest
is not None:
1052 manifest
= os
.path
.normpath(os
.path
.abspath(manifest
))
1053 if os
.path
.isfile(manifest
):
1054 return TestManifest([manifest
], strict
=True)
1056 toml_path
= os
.path
.join(manifest
, "xpcshell.toml")
1058 toml_path
= os
.path
.join(SCRIPT_DIR
, "tests", "xpcshell.toml")
1060 if os
.path
.exists(toml_path
):
1061 return TestManifest([toml_path
], strict
=True)
1064 "Failed to find manifest at %s; use --manifest "
1065 "to set path explicitly." % toml_path
1069 def normalizeTest(self
, root
, test_object
):
1070 path
= test_object
.get("file_relpath", test_object
["relpath"])
1071 if "dupe-manifest" in test_object
and "ancestor_manifest" in test_object
:
1072 test_object
["id"] = "%s:%s" % (
1073 os
.path
.basename(test_object
["ancestor_manifest"]),
1077 test_object
["id"] = path
1080 test_object
["manifest"] = os
.path
.relpath(test_object
["manifest"], root
)
1083 for key
in ("id", "manifest"):
1084 test_object
[key
] = test_object
[key
].replace(os
.sep
, "/")
1088 def buildTestList(self
, test_tags
=None, test_paths
=None, verify
=False):
1089 """Reads the xpcshell.toml manifest and set self.alltests to an array.
1091 Given the parameters, this method compiles a list of tests to be run
1092 that matches the criteria set by parameters.
1094 If any chunking of tests are to occur, it is also done in this method.
1096 If no tests are added to the list of tests to be run, an error
1097 is logged. A sys.exit() signal is sent to the caller.
1100 test_tags (list, optional): list of strings.
1101 test_paths (list, optional): list of strings derived from the command
1102 line argument provided by user, specifying
1104 verify (bool, optional): boolean value.
1106 if test_paths
is None:
1109 mp
= self
.getTestManifest(self
.manifest
)
1112 if build
and not root
:
1113 root
= build
.topsrcdir
1114 normalize
= partial(self
.normalizeTest
, root
)
1118 filters
.append(tags(test_tags
))
1122 path_filter
= pathprefix(test_paths
)
1123 filters
.append(path_filter
)
1125 noDefaultFilters
= False
1126 if self
.runFailures
:
1127 filters
.append(failures(self
.runFailures
))
1128 noDefaultFilters
= True
1130 if self
.totalChunks
> 1:
1131 filters
.append(chunk_by_slice(self
.thisChunk
, self
.totalChunks
))
1133 self
.alltests
= list(
1138 noDefaultFilters
=noDefaultFilters
,
1144 sys
.stderr
.write("*** offending mozinfo.info: %s\n" % repr(mozinfo
.info
))
1147 if path_filter
and path_filter
.missing
:
1149 "The following path(s) didn't resolve any tests:\n {}".format(
1150 " \n".join(sorted(path_filter
.missing
))
1154 if len(self
.alltests
) == 0:
1157 and path_filter
.missing
== set(test_paths
)
1158 and os
.environ
.get("MOZ_AUTOMATION") == "1"
1160 # This can happen in CI when a manifest doesn't exist due to a
1161 # build config variable in moz.build traversal. Don't generate
1162 # an error in this case. Adding a todo count avoids mozharness
1164 self
.todoCount
+= len(path_filter
.missing
)
1167 "no tests to run using specified "
1168 "combination of filters: {}".format(mp
.fmt_filters())
1172 if len(self
.alltests
) == 1 and not verify
:
1173 self
.singleFile
= os
.path
.basename(self
.alltests
[0]["path"])
1175 self
.singleFile
= None
1178 self
.dump_tests
= os
.path
.expanduser(self
.dump_tests
)
1179 assert os
.path
.exists(os
.path
.dirname(self
.dump_tests
))
1180 with
open(self
.dump_tests
, "w") as dumpFile
:
1181 dumpFile
.write(json
.dumps({"active_tests": self
.alltests
}))
1183 self
.log
.info("Dumping active_tests to %s file." % self
.dump_tests
)
1186 def setAbsPath(self
):
1188 Set the absolute path for xpcshell and xrepath. These 3 variables
1189 depend on input from the command line and we need to allow for absolute paths.
1190 This function is overloaded for a remote solution as os.path* won't work remotely.
1192 self
.testharnessdir
= os
.path
.dirname(os
.path
.abspath(__file__
))
1193 self
.headJSPath
= self
.testharnessdir
.replace("\\", "/") + "/head.js"
1194 if self
.xpcshell
is not None:
1195 self
.xpcshell
= os
.path
.abspath(self
.xpcshell
)
1197 if self
.app_binary
is not None:
1198 self
.app_binary
= os
.path
.abspath(self
.app_binary
)
1200 if self
.xrePath
is None:
1201 binary_path
= self
.app_binary
or self
.xpcshell
1202 self
.xrePath
= os
.path
.dirname(binary_path
)
1204 # Check if we're run from an OSX app bundle and override
1205 # self.xrePath if we are.
1206 appBundlePath
= os
.path
.join(
1207 os
.path
.dirname(os
.path
.dirname(self
.xpcshell
)), "Resources"
1209 if os
.path
.exists(os
.path
.join(appBundlePath
, "application.ini")):
1210 self
.xrePath
= appBundlePath
1212 self
.xrePath
= os
.path
.abspath(self
.xrePath
)
1214 if self
.mozInfo
is None:
1215 self
.mozInfo
= os
.path
.join(self
.testharnessdir
, "mozinfo.json")
1217 def buildPrefsFile(self
, extraPrefs
):
1218 # Create the prefs.js file
1220 # In test packages used in CI, the profile_data directory is installed
1221 # in the SCRIPT_DIR.
1222 profile_data_dir
= os
.path
.join(SCRIPT_DIR
, "profile_data")
1223 # If possible, read profile data from topsrcdir. This prevents us from
1224 # requiring a re-build to pick up newly added extensions in the
1225 # <profile>/extensions directory.
1227 path
= os
.path
.join(build
.topsrcdir
, "testing", "profiles")
1228 if os
.path
.isdir(path
):
1229 profile_data_dir
= path
1230 # Still not found? Look for testing/profiles relative to testing/xpcshell.
1231 if not os
.path
.isdir(profile_data_dir
):
1232 path
= os
.path
.abspath(os
.path
.join(SCRIPT_DIR
, "..", "profiles"))
1233 if os
.path
.isdir(path
):
1234 profile_data_dir
= path
1236 with
open(os
.path
.join(profile_data_dir
, "profiles.json"), "r") as fh
:
1237 base_profiles
= json
.load(fh
)["xpcshell"]
1239 # values to use when interpolating preferences
1241 "server": "dummyserver",
1244 profile
= Profile(profile
=self
.tempDir
, restore
=False)
1245 prefsFile
= os
.path
.join(profile
.profile
, "user.js")
1247 # Empty the user.js file in case the file existed before.
1248 with
open(prefsFile
, "w"):
1251 for name
in base_profiles
:
1252 path
= os
.path
.join(profile_data_dir
, name
)
1253 profile
.merge(path
, interpolation
=interpolation
)
1255 # add command line prefs
1256 prefs
= parse_preferences(extraPrefs
)
1257 profile
.set_preferences(prefs
)
1259 self
.prefsFile
= prefsFile
1262 def buildCoreEnvironment(self
):
1264 Add environment variables likely to be used across all platforms, including
1267 # Make assertions fatal
1268 self
.env
["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
1269 # Crash reporting interferes with debugging
1270 if not self
.debuggerInfo
:
1271 self
.env
["MOZ_CRASHREPORTER"] = "1"
1272 # Don't launch the crash reporter client
1273 self
.env
["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
1274 # Don't permit remote connections by default.
1275 # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
1276 # enable non-local connections for the purposes of local testing.
1277 # Don't override the user's choice here. See bug 1049688.
1278 self
.env
.setdefault("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "1")
1279 if self
.mozInfo
.get("topsrcdir") is not None:
1280 self
.env
["MOZ_DEVELOPER_REPO_DIR"] = self
.mozInfo
["topsrcdir"]
1281 if self
.mozInfo
.get("topobjdir") is not None:
1282 self
.env
["MOZ_DEVELOPER_OBJ_DIR"] = self
.mozInfo
["topobjdir"]
1284 # Disable the content process sandbox for the xpcshell tests. They
1285 # currently attempt to do things like bind() sockets, which is not
1286 # compatible with the sandbox.
1287 self
.env
["MOZ_DISABLE_CONTENT_SANDBOX"] = "1"
1288 if os
.getenv("MOZ_FETCHES_DIR", None):
1289 self
.env
["MOZ_FETCHES_DIR"] = os
.getenv("MOZ_FETCHES_DIR", None)
1291 if self
.mozInfo
.get("socketprocess_networking"):
1292 self
.env
["MOZ_FORCE_USE_SOCKET_PROCESS"] = "1"
1294 self
.env
["MOZ_DISABLE_SOCKET_PROCESS"] = "1"
1296 def buildEnvironment(self
):
1298 Create and returns a dictionary of self.env to include all the appropriate env
1299 variables and values. On a remote system, we overload this to set different
1300 values and are missing things like os.environ and PATH.
1302 self
.env
= dict(os
.environ
)
1303 self
.buildCoreEnvironment()
1304 if sys
.platform
== "win32":
1305 self
.env
["PATH"] = self
.env
["PATH"] + ";" + self
.xrePath
1306 elif sys
.platform
in ("os2emx", "os2knix"):
1307 os
.environ
["BEGINLIBPATH"] = self
.xrePath
+ ";" + self
.env
["BEGINLIBPATH"]
1308 os
.environ
["LIBPATHSTRICT"] = "T"
1309 elif sys
.platform
== "osx" or sys
.platform
== "darwin":
1310 self
.env
["DYLD_LIBRARY_PATH"] = os
.path
.join(
1311 os
.path
.dirname(self
.xrePath
), "MacOS"
1313 else: # unix or linux?
1314 if "LD_LIBRARY_PATH" not in self
.env
or self
.env
["LD_LIBRARY_PATH"] is None:
1315 self
.env
["LD_LIBRARY_PATH"] = self
.xrePath
1317 self
.env
["LD_LIBRARY_PATH"] = ":".join(
1318 [self
.xrePath
, self
.env
["LD_LIBRARY_PATH"]]
1321 usingASan
= "asan" in self
.mozInfo
and self
.mozInfo
["asan"]
1322 usingTSan
= "tsan" in self
.mozInfo
and self
.mozInfo
["tsan"]
1323 if usingASan
or usingTSan
:
1324 # symbolizer support
1325 if "ASAN_SYMBOLIZER_PATH" in self
.env
and os
.path
.isfile(
1326 self
.env
["ASAN_SYMBOLIZER_PATH"]
1328 llvmsym
= self
.env
["ASAN_SYMBOLIZER_PATH"]
1330 llvmsym
= os
.path
.join(
1331 self
.xrePath
, "llvm-symbolizer" + self
.mozInfo
["bin_suffix"]
1333 if os
.path
.isfile(llvmsym
):
1335 self
.env
["ASAN_SYMBOLIZER_PATH"] = llvmsym
1337 oldTSanOptions
= self
.env
.get("TSAN_OPTIONS", "")
1338 self
.env
["TSAN_OPTIONS"] = "external_symbolizer_path={} {}".format(
1339 llvmsym
, oldTSanOptions
1341 self
.log
.info("runxpcshelltests.py | using symbolizer at %s" % llvmsym
)
1344 "TEST-UNEXPECTED-FAIL | runxpcshelltests.py | "
1345 "Failed to find symbolizer at %s" % llvmsym
1352 Determine the value of the stdout and stderr for the test.
1353 Return value is a list (pStdout, pStderr).
1355 if self
.interactive
:
1359 if self
.debuggerInfo
and self
.debuggerInfo
.interactive
:
1363 if sys
.platform
== "os2emx":
1368 return pStdout
, pStderr
1370 def verifyDirPath(self
, dirname
):
1372 Simple wrapper to get the absolute path for a given directory name.
1373 On a remote system, we need to overload this to work on the remote filesystem.
1375 return os
.path
.abspath(dirname
)
1377 def trySetupNode(self
):
1379 Run node for HTTP/2 tests, if available, and updates mozinfo as appropriate.
1381 if os
.getenv("MOZ_ASSUME_NODE_RUNNING", None):
1382 self
.log
.info("Assuming required node servers are already running")
1383 if not os
.getenv("MOZHTTP2_PORT", None):
1385 "MOZHTTP2_PORT environment variable not set. "
1386 "Tests requiring http/2 will fail."
1390 # We try to find the node executable in the path given to us by the user in
1391 # the MOZ_NODE_PATH environment variable
1392 nodeBin
= os
.getenv("MOZ_NODE_PATH", None)
1393 if not nodeBin
and build
:
1394 nodeBin
= build
.substs
.get("NODEJS")
1397 "MOZ_NODE_PATH environment variable not set. "
1398 "Tests requiring http/2 will fail."
1402 if not os
.path
.exists(nodeBin
) or not os
.path
.isfile(nodeBin
):
1403 error
= "node not found at MOZ_NODE_PATH %s" % (nodeBin
)
1404 self
.log
.error(error
)
1405 raise IOError(error
)
1407 self
.log
.info("Found node at %s" % (nodeBin
,))
1409 def read_streams(name
, proc
, pipe
):
1410 output
= "stdout" if pipe
== proc
.stdout
else "stderr"
1411 for line
in iter(pipe
.readline
, ""):
1412 self
.log
.info("node %s [%s] %s" % (name
, output
, line
))
1414 def startServer(name
, serverJs
):
1415 if not os
.path
.exists(serverJs
):
1416 error
= "%s not found at %s" % (name
, serverJs
)
1417 self
.log
.error(error
)
1418 raise IOError(error
)
1420 # OK, we found our server, let's try to get it running
1421 self
.log
.info("Found %s at %s" % (name
, serverJs
))
1423 # We pipe stdin to node because the server will exit when its
1425 with
popenCleanupHack():
1427 [nodeBin
, serverJs
],
1433 universal_newlines
=True,
1434 start_new_session
=True,
1436 self
.nodeProc
[name
] = process
1438 # Check to make sure the server starts properly by waiting for it to
1439 # tell us it's started
1440 msg
= process
.stdout
.readline()
1441 if "server listening" in msg
:
1442 searchObj
= re
.search(
1443 r
"HTTP2 server listening on ports ([0-9]+),([0-9]+)", msg
, 0
1446 self
.env
["MOZHTTP2_PORT"] = searchObj
.group(1)
1447 self
.env
["MOZNODE_EXEC_PORT"] = searchObj
.group(2)
1449 target
=read_streams
,
1450 args
=(name
, process
, process
.stdout
),
1455 target
=read_streams
,
1456 args
=(name
, process
, process
.stderr
),
1460 except OSError as e
:
1461 # This occurs if the subprocess couldn't be started
1462 self
.log
.error("Could not run %s server: %s" % (name
, str(e
)))
1465 myDir
= os
.path
.split(os
.path
.abspath(__file__
))[0]
1466 startServer("moz-http2", os
.path
.join(myDir
, "moz-http2", "moz-http2.js"))
1468 def shutdownNode(self
):
1470 Shut down our node process, if it exists
1472 for name
, proc
in six
.iteritems(self
.nodeProc
):
1473 self
.log
.info("Node %s server shutting down ..." % name
)
1474 if proc
.poll() is not None:
1475 self
.log
.info("Node server %s already dead %s" % (name
, proc
.poll()))
1476 elif sys
.platform
!= "win32":
1477 # Kill process and all its spawned children.
1478 os
.killpg(proc
.pid
, signal
.SIGTERM
)
1484 def startHttp3Server(self
):
1486 Start a Http3 test server.
1489 if sys
.platform
== "win32":
1491 http3ServerPath
= self
.http3ServerPath
1492 if not http3ServerPath
:
1493 http3ServerPath
= os
.path
.join(
1494 SCRIPT_DIR
, "http3server", "http3server" + binSuffix
1497 http3ServerPath
= os
.path
.join(
1498 build
.topobjdir
, "dist", "bin", "http3server" + binSuffix
1500 dbPath
= os
.path
.join(SCRIPT_DIR
, "http3server", "http3serverDB")
1502 dbPath
= os
.path
.join(build
.topsrcdir
, "netwerk", "test", "http3serverDB")
1504 options
["http3ServerPath"] = http3ServerPath
1505 options
["profilePath"] = dbPath
1506 options
["isMochitest"] = False
1507 options
["isWin"] = sys
.platform
== "win32"
1508 serverEnv
= self
.env
.copy()
1509 serverLog
= self
.env
.get("MOZHTTP3_SERVER_LOG")
1510 if serverLog
is not None:
1511 serverEnv
["RUST_LOG"] = serverLog
1512 self
.http3Server
= Http3Server(options
, serverEnv
, self
.log
)
1513 self
.http3Server
.start()
1514 for key
, value
in self
.http3Server
.ports().items():
1515 self
.env
[key
] = value
1516 self
.env
["MOZHTTP3_ECH"] = self
.http3Server
.echConfig()
1518 def shutdownHttp3Server(self
):
1519 if self
.http3Server
is None:
1521 self
.http3Server
.stop()
1522 self
.http3Server
= None
1524 def buildXpcsRunArgs(self
):
1526 Add arguments to run the test or make it interactive.
1528 if self
.interactive
:
1529 self
.xpcsRunArgs
= [
1531 'print("To start the test, type |_execute_test();|.");',
1535 self
.xpcsRunArgs
= ["-e", "_execute_test(); quit(0);"]
1537 def addTestResults(self
, test
):
1538 self
.passCount
+= test
.passCount
1539 self
.failCount
+= test
.failCount
1540 self
.todoCount
+= test
.todoCount
1542 def updateMozinfo(self
, prefs
, options
):
1543 # Handle filenames in mozInfo
1544 if not isinstance(self
.mozInfo
, dict):
1545 mozInfoFile
= self
.mozInfo
1546 if not os
.path
.isfile(mozInfoFile
):
1548 "Error: couldn't find mozinfo.json at '%s'. Perhaps you "
1549 "need to use --build-info-json?" % mozInfoFile
1552 self
.mozInfo
= json
.load(open(mozInfoFile
))
1554 # mozinfo.info is used as kwargs. Some builds are done with
1555 # an older Python that can't handle Unicode keys in kwargs.
1556 # All of the keys in question should be ASCII.
1558 for k
, v
in self
.mozInfo
.items():
1559 if isinstance(k
, bytes
):
1560 k
= k
.decode("utf-8")
1562 self
.mozInfo
= fixedInfo
1564 self
.mozInfo
["fission"] = prefs
.get("fission.autostart", True)
1565 self
.mozInfo
["sessionHistoryInParent"] = self
.mozInfo
[
1567 ] or not prefs
.get("fission.disableSessionHistoryInParent", False)
1569 self
.mozInfo
["serviceworker_e10s"] = True
1571 self
.mozInfo
["verify"] = options
.get("verify", False)
1573 self
.mozInfo
["socketprocess_networking"] = prefs
.get(
1574 "network.http.network_access_on_socket_process.enabled", False
1577 self
.mozInfo
["condprof"] = options
.get("conditionedProfile", False)
1579 self
.mozInfo
["msix"] = options
.get(
1581 ) is not None and "WindowsApps" in options
.get("app_binary", "")
1583 self
.mozInfo
["is_ubuntu"] = "Ubuntu" in platform
.version()
1585 mozinfo
.update(self
.mozInfo
)
1590 def conditioned_profile_copy(self
):
1591 """Returns a copy of the original conditioned profile that was created."""
1592 condprof_copy
= os
.path
.join(tempfile
.mkdtemp(), "profile")
1594 self
.conditioned_profile_dir
,
1596 ignore
=shutil
.ignore_patterns("lock"),
1598 self
.log
.info("Created a conditioned-profile copy: %s" % condprof_copy
)
1599 return condprof_copy
1601 def downloadConditionedProfile(self
, profile_scenario
, app
):
1602 from condprof
.client
import get_profile
1603 from condprof
.util
import get_current_platform
, get_version
1605 if self
.conditioned_profile_dir
:
1606 # We already have a directory, so provide a copy that
1607 # will get deleted after it's done with
1608 return self
.conditioned_profile_dir
1610 # create a temp file to help ensure uniqueness
1611 temp_download_dir
= tempfile
.mkdtemp()
1613 "Making temp_download_dir from inside get_conditioned_profile {}".format(
1617 # call condprof's client API to yield our platform-specific
1618 # conditioned-profile binary
1619 platform
= get_current_platform()
1621 if isinstance(app
, str):
1622 version
= get_version(app
)
1624 if not profile_scenario
:
1625 profile_scenario
= "settled"
1627 cond_prof_target_dir
= get_profile(
1631 repo
="mozilla-central",
1637 # any other error is a showstopper
1638 self
.log
.critical("Could not get the conditioned profile")
1639 traceback
.print_exc()
1643 self
.log
.info("Retrying a profile with no version specified")
1644 cond_prof_target_dir
= get_profile(
1648 repo
="mozilla-central",
1652 self
.log
.critical("Could not get the conditioned profile")
1653 traceback
.print_exc()
1656 # now get the full directory path to our fetched conditioned profile
1657 self
.conditioned_profile_dir
= os
.path
.join(
1658 temp_download_dir
, cond_prof_target_dir
1660 if not os
.path
.exists(cond_prof_target_dir
):
1662 "Can't find target_dir {}, from get_profile()"
1663 "temp_download_dir {}, platform {}, scenario {}".format(
1664 cond_prof_target_dir
, temp_download_dir
, platform
, profile_scenario
1670 "Original self.conditioned_profile_dir is now set: {}".format(
1671 self
.conditioned_profile_dir
1674 return self
.conditioned_profile_copy
1676 def runSelfTest(self
):
1683 class XPCShellTestsTests(selftest
.XPCShellTestsTests
):
1684 def __init__(self
, name
):
1685 unittest
.TestCase
.__init
__(self
, name
)
1686 self
.testing_modules
= this
.testingModulesDir
1687 self
.xpcshellBin
= this
.xpcshell
1688 self
.app_binary
= this
.app_binary
1689 self
.utility_path
= this
.utility_path
1690 self
.symbols_path
= this
.symbolsPath
1692 old_info
= dict(mozinfo
.info
)
1694 suite
= unittest
.TestLoader().loadTestsFromTestCase(XPCShellTestsTests
)
1695 return unittest
.TextTestRunner(verbosity
=2).run(suite
).wasSuccessful()
1697 # The self tests modify mozinfo, so we need to reset it.
1698 mozinfo
.info
.clear()
1699 mozinfo
.update(old_info
)
1701 def runTests(self
, options
, testClass
=XPCShellTestThread
, mobileArgs
=None):
1707 # Number of times to repeat test(s) in --verify mode
1710 if isinstance(options
, Namespace
):
1711 options
= vars(options
)
1713 # Try to guess modules directory.
1714 # This somewhat grotesque hack allows the buildbot machines to find the
1715 # modules directory without having to configure the buildbot hosts. This
1716 # code path should never be executed in local runs because the build system
1717 # should always set this argument.
1718 if not options
.get("testingModulesDir"):
1719 possible
= os
.path
.join(here
, os
.path
.pardir
, "modules")
1721 if os
.path
.isdir(possible
):
1722 testingModulesDir
= possible
1724 if options
.get("rerun_failures"):
1725 if os
.path
.exists(options
.get("failure_manifest")):
1726 rerun_manifest
= os
.path
.join(
1727 os
.path
.dirname(options
["failure_manifest"]), "rerun.toml"
1729 shutil
.copyfile(options
["failure_manifest"], rerun_manifest
)
1730 os
.remove(options
["failure_manifest"])
1732 self
.log
.error("No failures were found to re-run.")
1735 if options
.get("testingModulesDir"):
1736 # The resource loader expects native paths. Depending on how we were
1737 # invoked, a UNIX style path may sneak in on Windows. We try to
1739 testingModulesDir
= os
.path
.normpath(options
["testingModulesDir"])
1741 if not os
.path
.isabs(testingModulesDir
):
1742 testingModulesDir
= os
.path
.abspath(testingModulesDir
)
1744 if not testingModulesDir
.endswith(os
.path
.sep
):
1745 testingModulesDir
+= os
.path
.sep
1747 self
.debuggerInfo
= None
1749 if options
.get("debugger"):
1750 self
.debuggerInfo
= mozdebug
.get_debugger_info(
1751 options
.get("debugger"),
1752 options
.get("debuggerArgs"),
1753 options
.get("debuggerInteractive"),
1756 self
.jsDebuggerInfo
= None
1757 if options
.get("jsDebugger"):
1758 # A namedtuple let's us keep .port instead of ['port']
1759 JSDebuggerInfo
= namedtuple("JSDebuggerInfo", ["port"])
1760 self
.jsDebuggerInfo
= JSDebuggerInfo(port
=options
["jsDebuggerPort"])
1762 self
.app_binary
= options
.get("app_binary")
1763 self
.xpcshell
= options
.get("xpcshell")
1764 self
.http3ServerPath
= options
.get("http3server")
1765 self
.xrePath
= options
.get("xrePath")
1766 self
.utility_path
= options
.get("utility_path")
1767 self
.appPath
= options
.get("appPath")
1768 self
.symbolsPath
= options
.get("symbolsPath")
1769 self
.tempDir
= os
.path
.normpath(options
.get("tempDir") or tempfile
.gettempdir())
1770 self
.manifest
= options
.get("manifest")
1771 self
.dump_tests
= options
.get("dump_tests")
1772 self
.interactive
= options
.get("interactive")
1773 self
.verbose
= options
.get("verbose")
1774 self
.verboseIfFails
= options
.get("verboseIfFails")
1775 self
.keepGoing
= options
.get("keepGoing")
1776 self
.logfiles
= options
.get("logfiles")
1777 self
.totalChunks
= options
.get("totalChunks", 1)
1778 self
.thisChunk
= options
.get("thisChunk")
1779 self
.profileName
= options
.get("profileName") or "xpcshell"
1780 self
.mozInfo
= options
.get("mozInfo")
1781 self
.testingModulesDir
= testingModulesDir
1782 self
.sequential
= options
.get("sequential")
1783 self
.failure_manifest
= options
.get("failure_manifest")
1784 self
.threadCount
= options
.get("threadCount") or NUM_THREADS
1785 self
.jscovdir
= options
.get("jscovdir")
1786 self
.headless
= options
.get("headless")
1787 self
.runFailures
= options
.get("runFailures")
1788 self
.timeoutAsPass
= options
.get("timeoutAsPass")
1789 self
.crashAsPass
= options
.get("crashAsPass")
1790 self
.conditionedProfile
= options
.get("conditionedProfile")
1791 self
.repeat
= options
.get("repeat", 0)
1798 if self
.conditionedProfile
:
1799 self
.conditioned_profile_dir
= self
.downloadConditionedProfile(
1800 "full", self
.appPath
1802 options
["self_test"] = False
1803 if not options
["test_tags"]:
1804 options
["test_tags"] = []
1805 options
["test_tags"].append("condprof")
1809 eprefs
= options
.get("extraPrefs") or []
1810 # enable fission by default
1811 if options
.get("disableFission"):
1812 eprefs
.append("fission.autostart=false")
1814 # should be by default, just in case
1815 eprefs
.append("fission.autostart=true")
1817 prefs
= self
.buildPrefsFile(eprefs
)
1818 self
.buildXpcsRunArgs()
1820 self
.event
= Event()
1822 if not self
.updateMozinfo(prefs
, options
):
1826 "These variables are available in the mozinfo environment and "
1827 "can be used to skip tests conditionally:"
1829 for info
in sorted(self
.mozInfo
.items(), key
=lambda item
: item
[0]):
1830 self
.log
.info(" {key}: {value}".format(key
=info
[0], value
=info
[1]))
1832 if options
.get("self_test"):
1833 if not self
.runSelfTest():
1837 ("tsan" in self
.mozInfo
and self
.mozInfo
["tsan"])
1838 or ("asan" in self
.mozInfo
and self
.mozInfo
["asan"])
1839 ) and not options
.get("threadCount"):
1840 # TSan/ASan require significantly more memory, so reduce the amount of parallel
1841 # tests we run to avoid OOMs and timeouts. We always keep a minimum of 2 for
1842 # non-sequential execution.
1843 # pylint --py3k W1619
1844 self
.threadCount
= max(self
.threadCount
/ 2, 2)
1846 self
.stack_fixer_function
= None
1847 if self
.utility_path
and os
.path
.exists(self
.utility_path
):
1848 self
.stack_fixer_function
= get_stack_fixer_function(
1849 self
.utility_path
, self
.symbolsPath
1852 # buildEnvironment() needs mozInfo, so we call it after mozInfo is initialized.
1853 self
.buildEnvironment()
1855 # The appDirKey is a optional entry in either the default or individual test
1856 # sections that defines a relative application directory for test runs. If
1857 # defined we pass 'grePath/$appDirKey' for the -a parameter of the xpcshell
1860 if "appname" in self
.mozInfo
:
1861 appDirKey
= self
.mozInfo
["appname"] + "-appdir"
1863 # We have to do this before we run tests that depend on having the node
1867 self
.startHttp3Server()
1869 pStdout
, pStderr
= self
.getPipes()
1872 options
.get("test_tags"), options
.get("testPaths"), options
.get("verify")
1875 self
.sequential
= True
1877 if options
.get("shuffle"):
1878 random
.shuffle(self
.alltests
)
1880 self
.cleanup_dir_list
= []
1883 "appPath": self
.appPath
,
1884 "xrePath": self
.xrePath
,
1885 "utility_path": self
.utility_path
,
1886 "testingModulesDir": self
.testingModulesDir
,
1887 "debuggerInfo": self
.debuggerInfo
,
1888 "jsDebuggerInfo": self
.jsDebuggerInfo
,
1889 "headJSPath": self
.headJSPath
,
1890 "tempDir": self
.tempDir
,
1891 "testharnessdir": self
.testharnessdir
,
1892 "profileName": self
.profileName
,
1893 "singleFile": self
.singleFile
,
1894 "env": self
.env
, # making a copy of this in the testthreads
1895 "symbolsPath": self
.symbolsPath
,
1896 "logfiles": self
.logfiles
,
1897 "app_binary": self
.app_binary
,
1898 "xpcshell": self
.xpcshell
,
1899 "xpcsRunArgs": self
.xpcsRunArgs
,
1900 "failureManifest": self
.failure_manifest
,
1901 "jscovdir": self
.jscovdir
,
1902 "harness_timeout": self
.harness_timeout
,
1903 "stack_fixer_function": self
.stack_fixer_function
,
1904 "event": self
.event
,
1905 "cleanup_dir_list": self
.cleanup_dir_list
,
1908 "keep_going": self
.keepGoing
,
1910 "interactive": self
.interactive
,
1911 "app_dir_key": appDirKey
,
1912 "rootPrefsFile": self
.prefsFile
,
1913 "extraPrefs": options
.get("extraPrefs") or [],
1914 "verboseIfFails": self
.verboseIfFails
,
1915 "headless": self
.headless
,
1916 "runFailures": self
.runFailures
,
1917 "timeoutAsPass": self
.timeoutAsPass
,
1918 "crashAsPass": self
.crashAsPass
,
1919 "conditionedProfileDir": self
.conditioned_profile_dir
,
1920 "repeat": self
.repeat
,
1924 # Allow user to kill hung xpcshell subprocess with SIGINT
1925 # when we are only running tests sequentially.
1926 signal
.signal(signal
.SIGINT
, markGotSIGINT
)
1928 if self
.debuggerInfo
:
1929 # Force a sequential run
1930 self
.sequential
= True
1932 # If we have an interactive debugger, disable SIGINT entirely.
1933 if self
.debuggerInfo
.interactive
:
1934 signal
.signal(signal
.SIGINT
, lambda signum
, frame
: None)
1936 if "lldb" in self
.debuggerInfo
.path
:
1937 # Ask people to start debugging using 'process launch', see bug 952211.
1939 "It appears that you're using LLDB to debug this test. "
1940 + "Please use the 'process launch' command instead of "
1941 "the 'run' command to start xpcshell."
1944 if self
.jsDebuggerInfo
:
1945 # The js debugger magic needs more work to do the right thing
1946 # if debugging multiple files.
1947 if len(self
.alltests
) != 1:
1949 "Error: --jsdebugger can only be used with a single test!"
1953 # The test itself needs to know whether it is a tsan build, since
1954 # that has an effect on interpretation of the process return value.
1955 usingTSan
= "tsan" in self
.mozInfo
and self
.mozInfo
["tsan"]
1957 usingCrashReporter
= (
1958 "crashreporter" in self
.mozInfo
and self
.mozInfo
["crashreporter"]
1961 # create a queue of all tests that will run
1962 tests_queue
= deque()
1963 # also a list for the tests that need to be run sequentially
1964 sequential_tests
= []
1967 if options
.get("repeat", 0) > 0:
1968 self
.sequential
= True
1970 if not options
.get("verify"):
1971 for test_object
in self
.alltests
:
1972 # Test identifiers are provided for the convenience of logging. These
1973 # start as path names but are rewritten in case tests from the same path
1976 path
= test_object
["path"]
1978 if self
.singleFile
and not path
.endswith(self
.singleFile
):
1981 # if we have --repeat, duplicate the tests as needed
1982 for i
in range(0, options
.get("repeat", 0) + 1):
1987 verbose
=self
.verbose
or test_object
.get("verbose") == "true",
1988 usingTSan
=usingTSan
,
1989 usingCrashReporter
=usingCrashReporter
,
1990 mobileArgs
=mobileArgs
,
1993 if "run-sequentially" in test_object
or self
.sequential
:
1994 sequential_tests
.append(test
)
1996 tests_queue
.append(test
)
1998 status
= self
.runTestList(
1999 tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2003 # Test verification: Run each test many times, in various configurations,
2004 # in hopes of finding intermittent failures.
2008 # Run tests sequentially. Parallel mode would also work, except that
2009 # the logging system gets confused when 2 or more tests with the same
2010 # name run at the same time.
2011 sequential_tests
= []
2012 for i
in range(VERIFY_REPEAT
):
2015 test_object
, retry
=False, mobileArgs
=mobileArgs
, **kwargs
2017 sequential_tests
.append(test
)
2018 status
= self
.runTestList(
2019 tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2024 # Run tests sequentially, with MOZ_CHAOSMODE enabled.
2025 sequential_tests
= []
2026 self
.env
["MOZ_CHAOSMODE"] = "0xfb"
2027 # chaosmode runs really slow, allow tests extra time to pass
2028 self
.harness_timeout
= self
.harness_timeout
* 2
2029 for i
in range(VERIFY_REPEAT
):
2032 test_object
, retry
=False, mobileArgs
=mobileArgs
, **kwargs
2034 sequential_tests
.append(test
)
2035 status
= self
.runTestList(
2036 tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2038 self
.harness_timeout
= self
.harness_timeout
/ 2
2042 ("1. Run each test %d times, sequentially." % VERIFY_REPEAT
, step1
),
2044 "2. Run each test %d times, sequentially, in chaos mode."
2049 startTime
= datetime
.now()
2050 maxTime
= timedelta(seconds
=options
["verifyMaxTime"])
2051 for test_object
in self
.alltests
:
2053 for descr
, step
in steps
:
2054 stepResults
[descr
] = "not run / incomplete"
2055 finalResult
= "PASSED"
2056 for descr
, step
in steps
:
2057 if (datetime
.now() - startTime
) > maxTime
:
2059 "::: Test verification is taking too long: Giving up!"
2062 "::: So far, all checks passed, but not "
2063 "all checks were run."
2066 self
.log
.info(":::")
2067 self
.log
.info('::: Running test verification step "%s"...' % descr
)
2068 self
.log
.info(":::")
2070 if status
is not True:
2071 stepResults
[descr
] = "FAIL"
2072 finalResult
= "FAILED!"
2074 stepResults
[descr
] = "Pass"
2075 self
.log
.info(":::")
2077 "::: Test verification summary for: %s" % test_object
["path"]
2079 self
.log
.info(":::")
2080 for descr
in sorted(stepResults
.keys()):
2081 self
.log
.info("::: %s : %s" % (descr
, stepResults
[descr
]))
2082 self
.log
.info(":::")
2083 self
.log
.info("::: Test verification %s" % finalResult
)
2084 self
.log
.info(":::")
2087 self
.shutdownHttp3Server()
2091 def start_test(self
, test
):
2094 def test_ended(self
, test
):
2098 self
, tests_queue
, sequential_tests
, testClass
, mobileArgs
, **kwargs
2101 self
.log
.info("Running tests sequentially.")
2103 self
.log
.info("Using at most %d threads." % self
.threadCount
)
2105 # keep a set of threadCount running tests and start running the
2106 # tests in the queue at most threadCount at a time
2107 running_tests
= set()
2112 self
.try_again_list
= []
2114 tests_by_manifest
= defaultdict(list)
2115 for test
in self
.alltests
:
2116 group
= get_full_group_name(test
)
2117 tests_by_manifest
[group
].append(test
["id"])
2119 self
.log
.suite_start(tests_by_manifest
, name
="xpcshell")
2121 while tests_queue
or running_tests
:
2122 # if we're not supposed to continue and all of the running tests
2124 if not keep_going
and not running_tests
:
2127 # if there's room to run more tests, start running them
2129 keep_going
and tests_queue
and (len(running_tests
) < self
.threadCount
)
2131 test
= tests_queue
.popleft()
2132 running_tests
.add(test
)
2133 self
.start_test(test
)
2135 # queue is full (for now) or no more new tests,
2136 # process the finished tests so far
2138 # wait for at least one of the tests to finish
2142 # find what tests are done (might be more than 1)
2144 for test
in running_tests
:
2146 self
.test_ended(test
)
2147 done_tests
.add(test
)
2150 ) # join with timeout so we don't hang on blocked threads
2151 # if the test had trouble, we will try running it again
2152 # at the end of the run
2153 if test
.retry
or test
.is_alive():
2154 # if the join call timed out, test.is_alive => True
2155 self
.try_again_list
.append(test
.test_object
)
2157 # did the test encounter any exception?
2159 exceptions
.append(test
.exception
)
2160 tracebacks
.append(test
.traceback
)
2161 # we won't add any more tests, will just wait for
2162 # the currently running ones to finish
2164 infra_abort
= infra_abort
and test
.infra
2165 keep_going
= keep_going
and test
.keep_going
2166 self
.addTestResults(test
)
2168 # make room for new tests to run
2169 running_tests
.difference_update(done_tests
)
2172 return TBPL_RETRY
# terminate early
2175 # run the other tests sequentially
2176 for test
in sequential_tests
:
2179 "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so "
2180 "stopped run. (Use --keep-going to keep running tests "
2181 "after killing one with SIGINT)"
2184 self
.start_test(test
)
2186 self
.test_ended(test
)
2187 if (test
.failCount
> 0 or test
.passCount
<= 0) and os
.environ
.get(
2190 self
.try_again_list
.append(test
.test_object
)
2192 self
.addTestResults(test
)
2193 # did the test encounter any exception?
2195 exceptions
.append(test
.exception
)
2196 tracebacks
.append(test
.traceback
)
2198 keep_going
= test
.keep_going
2200 # retry tests that failed when run in parallel
2201 if self
.try_again_list
:
2202 self
.log
.info("Retrying tests that failed when run in parallel.")
2203 for test_object
in self
.try_again_list
:
2207 verbose
=self
.verbose
,
2208 mobileArgs
=mobileArgs
,
2211 self
.start_test(test
)
2213 self
.test_ended(test
)
2214 self
.addTestResults(test
)
2215 # did the test encounter any exception?
2217 exceptions
.append(test
.exception
)
2218 tracebacks
.append(test
.traceback
)
2220 keep_going
= test
.keep_going
2222 # restore default SIGINT behaviour
2223 signal
.signal(signal
.SIGINT
, signal
.SIG_DFL
)
2225 # Clean up any slacker directories that might be lying around
2226 # Some might fail because of windows taking too long to unlock them.
2227 # We don't do anything if this fails because the test machines will have
2228 # their $TEMP dirs cleaned up on reboot anyway.
2229 for directory
in self
.cleanup_dir_list
:
2231 shutil
.rmtree(directory
)
2233 self
.log
.info("%s could not be cleaned up." % directory
)
2236 self
.log
.info("Following exceptions were raised:")
2237 for t
in tracebacks
:
2241 if self
.testCount
== 0 and os
.environ
.get("MOZ_AUTOMATION") != "1":
2242 self
.log
.error("No tests run. Did you pass an invalid --test-path?")
2245 # doing this allows us to pass the mozharness parsers that
2246 # report an orange job for failCount>0
2247 if self
.runFailures
:
2248 passed
= self
.passCount
2249 self
.passCount
= self
.failCount
2250 self
.failCount
= passed
2252 self
.log
.info("INFO | Result summary:")
2253 self
.log
.info("INFO | Passed: %d" % self
.passCount
)
2254 self
.log
.info("INFO | Failed: %d" % self
.failCount
)
2255 self
.log
.info("INFO | Todo: %d" % self
.todoCount
)
2256 self
.log
.info("INFO | Retried: %d" % len(self
.try_again_list
))
2258 if gotSIGINT
and not keep_going
:
2260 "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. "
2261 "(Use --keep-going to keep running tests after "
2262 "killing one with SIGINT)"
2266 self
.log
.suite_end()
2267 return self
.runFailures
or self
.failCount
== 0
2271 parser
= parser_desktop()
2272 options
= parser
.parse_args()
2274 log
= commandline
.setup_logging("XPCShell", options
, {"tbpl": sys
.stdout
})
2276 if options
.xpcshell
is None and options
.app_binary
is None:
2278 "Must provide path to xpcshell using --xpcshell or Firefox using --app-binary"
2282 if options
.xpcshell
is not None and options
.app_binary
is not None:
2284 "Cannot provide --xpcshell and --app-binary - they are mutually exclusive options. Choose one."
2288 xpcsh
= XPCShellTests(log
)
2290 if options
.interactive
and not options
.testPath
:
2291 log
.error("Error: You must specify a test filename in interactive mode!")
2294 result
= xpcsh
.runTests(options
)
2295 if result
== TBPL_RETRY
:
2302 if __name__
== "__main__":