2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
11 sys.path.append(Dir('#/tools').abspath)
15 Import(['pre_base_env'])
17 # Underlay things migrating to ppapi repo.
18 Dir('#/..').addRepository(Dir('#/../ppapi'))
20 # Load a config file from the Chrome repo. This allows scons files to be added
21 # and removed in Chrome without requiring a DEPS roll.
22 ppapi_scons_files = {}
24 File('#/../ppapi/native_client/ppapi_scons_files.py').abspath,
28 # Append a list of files to another, filtering out the files that already exist.
29 # Filtering helps migrate declarations between repos by preventing redundant
30 # declarations from causing an error.
31 def ExtendFileList(existing, additional):
32 # Avoid quadratic behavior by using a set.
34 for file_name in existing + additional:
35 if file_name in combined:
36 print 'WARNING: two references to file %s in the build.' % file_name
37 combined.add(file_name)
38 return sorted(combined)
40 ppapi_scons_files['nonvariant_test_scons_files'] = ExtendFileList(
41 ppapi_scons_files.get('nonvariant_test_scons_files', []), [
42 'tests/nacl_browser/browser_dynamic_library/nacl.scons',
43 'tests/nacl_browser/browser_startup_time/nacl.scons',
44 'tests/nacl_browser/inbrowser_test_runner/nacl.scons',
45 'tests/nacl_browser/manifest_file/nacl.scons',
46 'tests/nacl_browser/nameservice/nacl.scons',
47 'tests/nacl_browser/postmessage_redir/nacl.scons',
50 ppapi_scons_files['irt_variant_test_scons_files'] = ExtendFileList(
51 ppapi_scons_files.get('irt_variant_test_scons_files', []), [
52 # Disabled by Brad Chen 4 Sep to try to green Chromium
53 # nacl_integration tests
54 #'tests/nacl_browser/fault_injection/nacl.scons',
55 'tests/nacl_browser/pnacl_client_translator/nacl.scons',
58 ppapi_scons_files['untrusted_scons_files'] = ExtendFileList(
59 ppapi_scons_files.get('untrusted_scons_files', []), [
60 'src/untrusted/irt_stub/nacl.scons',
61 'src/untrusted/nacl_ppapi_util/nacl.scons',
62 'src/untrusted/pnacl_irt_shim/nacl.scons',
63 'src/untrusted/pnacl_support_extension/nacl.scons',
66 EXTRA_ENV = ['XAUTHORITY', 'HOME', 'DISPLAY', 'SSH_TTY', 'KRB5CCNAME']
68 def SetupBrowserEnv(env):
69 for var_name in EXTRA_ENV:
70 if var_name in os.environ:
71 env['ENV'][var_name] = os.environ[var_name]
73 pre_base_env.AddMethod(SetupBrowserEnv)
76 def GetHeadlessPrefix(env):
77 if env.Bit('browser_headless') and env.Bit('host_linux'):
78 return ['xvfb-run', '--auto-servernum']
80 # Mac and Windows do not seem to have an equivalent.
83 pre_base_env.AddMethod(GetHeadlessPrefix)
86 # A fake file to depend on if a path to Chrome is not specified.
87 no_browser = pre_base_env.File('chrome_browser_path_not_specified')
90 # SCons attempts to run a test that depends on "no_browser", detect this at
91 # runtime and cause a build error.
92 def NoBrowserError(target, source, env):
93 print target, source, env
94 print ("***\nYou need to specificy chrome_browser_path=... on the " +
95 "command line to run these tests.\n***\n")
98 pre_base_env.Append(BUILDERS = {
99 'NoBrowserError': Builder(action=NoBrowserError)
102 pre_base_env.NoBrowserError([no_browser], [])
105 def ChromeBinary(env):
106 if 'chrome_browser_path' in ARGUMENTS:
107 return env.File(env.SConstructAbsPath(ARGUMENTS['chrome_browser_path']))
111 pre_base_env.AddMethod(ChromeBinary)
114 def GetPPAPIPluginPath(env, allow_64bit_redirect=True):
115 if 'force_ppapi_plugin' in ARGUMENTS:
116 return env.SConstructAbsPath(ARGUMENTS['force_ppapi_plugin'])
118 fn = env.File('${STAGING_DIR}/ppNaClPlugin')
120 fn = env.File('${STAGING_DIR}/${SHLIBPREFIX}ppNaClPlugin${SHLIBSUFFIX}')
121 if allow_64bit_redirect and env.Bit('target_x86_64'):
122 # On 64-bit Windows and on Mac, we need the 32-bit plugin because
123 # the browser is 32-bit.
124 # Unfortunately it is tricky to build the 32-bit plugin (and all the
125 # libraries it needs) in a 64-bit build... so we'll assume it has already
126 # been built in a previous invocation.
127 # TODO(ncbray) better 32/64 builds.
128 if env.Bit('windows'):
129 fn = env.subst(fn).abspath.replace('-win-x86-64', '-win-x86-32')
131 fn = env.subst(fn).abspath.replace('-mac-x86-64', '-mac-x86-32')
134 pre_base_env.AddMethod(GetPPAPIPluginPath)
137 # runnable-ld.so log has following format:
138 # lib_name => path_to_lib (0x....address)
139 def ParseLibInfoInRunnableLdLog(line):
140 pos = line.find(' => ')
143 lib_name = line[:pos].strip()
144 lib_path = line[pos+4:]
145 pos1 = lib_path.rfind(' (')
148 lib_path = lib_path[:pos1]
149 return lib_name, lib_path
152 # Expected name of the temporary .libs file which stores glibc library
153 # dependencies in "lib_name => lib_info" format
154 # (see ParseLibInfoInRunnableLdLog)
155 def GlibcManifestLibsListFilename(manifest_base_name):
156 return '${STAGING_DIR}/%s.libs' % manifest_base_name
159 # Copy libs and manifest to the target directory.
160 # source[0] is a manifest file
161 # source[1] is a .libs file with a list of libs generated by runnable-ld.so
162 def CopyLibsForExtensionCommand(target, source, env):
163 source_manifest = str(source[0])
164 target_manifest = str(target[0])
165 shutil.copyfile(source_manifest, target_manifest)
166 target_dir = os.path.dirname(target_manifest)
167 libs_file = open(str(source[1]), 'r')
168 for line in libs_file.readlines():
169 lib_info = ParseLibInfoInRunnableLdLog(line)
171 lib_name, lib_path = lib_info
172 # Note: This probably does NOT work with pnacl _pexes_ right now, because
173 # the NEEDED metadata in the bitcode doesn't have the original file paths.
174 # This should probably be done without such knowledge.
175 if lib_path == 'NaClMain':
176 # This is a fake file name, which we cannot copy.
178 shutil.copyfile(lib_path, os.path.join(target_dir, lib_name))
179 shutil.copyfile(env.subst('${NACL_SDK_LIB}/runnable-ld.so'),
180 os.path.join(target_dir, 'runnable-ld.so'))
184 # Extensions are loaded from directory on disk and so all dynamic libraries
185 # they use must be copied to extension directory. The option --extra_serving_dir
186 # does not help us in this case.
187 def CopyLibsForExtension(env, target_dir, manifest):
188 if not env.Bit('nacl_glibc'):
189 return env.Install(target_dir, manifest)
190 manifest_base_name = os.path.basename(str(env.subst(manifest)))
191 lib_list_node = env.File(GlibcManifestLibsListFilename(manifest_base_name))
192 nmf_node = env.Command(
193 target_dir + '/' + manifest_base_name,
194 [manifest, lib_list_node],
195 CopyLibsForExtensionCommand)
198 pre_base_env.AddMethod(CopyLibsForExtension)
202 def WhitelistLibsForExtensionCommand(target, source, env):
203 # Load existing extension manifest.
204 src_file = open(source[0].abspath, 'r')
205 src_json = json.load(src_file)
208 # Load existing 'web_accessible_resources' key.
209 if 'web_accessible_resources' not in src_json:
210 src_json['web_accessible_resources'] = []
211 web_accessible = src_json['web_accessible_resources']
213 # Load list of libraries, and add libraries to web_accessible list.
214 libs_file = open(source[1].abspath, 'r')
215 for line in libs_file.readlines():
216 lib_info = ParseLibInfoInRunnableLdLog(line)
218 web_accessible.append(lib_info[0])
219 # Also add the dynamic loader, which won't be in the libs_file.
220 web_accessible.append('runnable-ld.so')
223 # Write out the appended-to extension manifest.
224 target_file = open(target[0].abspath, 'w')
225 json.dump(src_json, target_file, sort_keys=True, indent=2)
229 # Whitelist glibc shared libraries (if necessary), so that they are
230 # 'web_accessible_resources'. This allows the libraries hosted at the origin
231 # chrome-extension://[PACKAGE ID]/
232 # to be made available to webpages that use this NaCl extension,
233 # which are in a different origin.
234 # See: http://code.google.com/chrome/extensions/manifest.html
236 # Alternatively, we could try to use the chrome commandline switch
237 # '--disable-extensions-resource-whitelist', but that would not be what
238 # users will need to do.
239 def WhitelistLibsForExtension(env, target_dir, nmf, extension_manifest):
240 if env.Bit('nacl_static_link'):
241 # For static linking, assume the nexe and nmf files are already
242 # whitelisted, so there is no need to add entries to the extension_manifest.
243 return env.Install(target_dir, extension_manifest)
244 nmf_base_name = os.path.basename(env.File(nmf).abspath)
245 lib_list_node = env.File(GlibcManifestLibsListFilename(nmf_base_name))
246 manifest_base_name = os.path.basename(env.File(extension_manifest).abspath)
247 extension_manifest_node = env.Command(
248 target_dir + '/' + manifest_base_name,
249 [extension_manifest, lib_list_node],
250 WhitelistLibsForExtensionCommand)
251 return extension_manifest_node
253 pre_base_env.AddMethod(WhitelistLibsForExtension)
256 # Generate manifest from newlib manifest and the list of libs generated by
258 def GenerateManifestFunc(target, source, env):
259 # Open the original manifest and parse it.
260 source_file = open(str(source[0]), 'r')
261 obj = json.load(source_file)
263 # Open the file with ldd-format list of NEEDED libs and parse it.
264 libs_file = open(str(source[1]), 'r')
266 arch = env.subst('${TARGET_FULLARCH}')
267 for line in libs_file.readlines():
268 lib_info = ParseLibInfoInRunnableLdLog(line)
270 lib_name, _ = lib_info
271 lib_names.append(lib_name)
273 # Inject the NEEDED libs into the manifest.
274 if 'files' not in obj:
276 for lib_name in lib_names:
277 obj['files'][lib_name] = {}
278 obj['files'][lib_name][arch] = {}
279 obj['files'][lib_name][arch]['url'] = lib_name
280 # Put what used to be specified under 'program' into 'main.nexe'.
281 obj['files']['main.nexe'] = {}
282 for k, v in obj['program'].items():
283 obj['files']['main.nexe'][k] = v.copy()
284 v['url'] = 'runnable-ld.so'
285 # Write the new manifest!
286 target_file = open(str(target[0]), 'w')
287 json.dump(obj, target_file, sort_keys=True, indent=2)
291 def GenerateManifestPnacl(env, dest_file, manifest, exe_file):
294 ['${GENNMF}', exe_file, manifest],
295 # Generate a flat url scheme to simplify file-staging.
296 ('${SOURCES[0]} ${SOURCES[1]} -L${NACL_SDK_LIB} -L${LIB_DIR} '
297 ' --flat-url-scheme --base-nmf ${SOURCES[2]} -o ${TARGET}'))
299 def GenerateManifestDynamicLink(env, dest_file, lib_list_file,
301 # Run sel_ldr on the nexe to trace the NEEDED libraries.
302 lib_list_node = env.Command(
305 '${NACL_SDK_LIB}/runnable-ld.so',
307 '${SCONSTRUCT_DIR}/DEPS'],
308 # We ignore the return code using '-' in order to build tests
309 # where binaries do not validate. This is a Scons feature.
310 '-${SOURCES[0]} -a -E LD_TRACE_LOADED_OBJECTS=1 ${SOURCES[1]} '
311 '--library-path ${NACL_SDK_LIB}:${LIB_DIR} ${SOURCES[2].posix} '
313 return env.Command(dest_file,
314 [manifest, lib_list_node],
315 GenerateManifestFunc)[0]
318 def GenerateSimpleManifestStaticLink(env, dest_file, exe_name):
319 def Func(target, source, env):
320 archs = ('x86-32', 'x86-64', 'arm')
321 nmf_data = {'program': dict((arch, {'url': '%s_%s.nexe' % (exe_name, arch)})
323 fh = open(target[0].abspath, 'w')
324 json.dump(nmf_data, fh, sort_keys=True, indent=2)
326 node = env.Command(dest_file, [], Func)[0]
327 # Scons does not track the dependency of dest_file on exe_name or on
328 # the Python code above, so we should always recreate dest_file when
330 env.AlwaysBuild(node)
334 def GenerateSimpleManifest(env, dest_file, exe_name):
335 if env.Bit('pnacl_generate_pexe'):
336 static_manifest = GenerateSimpleManifestStaticLink(
337 env, '%s.static' % dest_file, exe_name)
338 return GenerateManifestPnacl(env, dest_file, static_manifest,
339 '${STAGING_DIR}/%s.pexe' %
340 env.ProgramNameForNmf(exe_name))
341 elif env.Bit('nacl_static_link'):
342 return GenerateSimpleManifestStaticLink(env, dest_file, exe_name)
344 static_manifest = GenerateSimpleManifestStaticLink(
345 env, '%s.static' % dest_file, exe_name)
346 return GenerateManifestDynamicLink(
347 env, dest_file, '%s.tmp_lib_list' % dest_file, static_manifest,
348 '${STAGING_DIR}/%s.nexe' % env.ProgramNameForNmf(exe_name))
350 pre_base_env.AddMethod(GenerateSimpleManifest)
353 # Returns a pair (main program, is_portable), based on the program
354 # specified in manifest file.
355 def GetMainProgramFromManifest(env, manifest):
356 obj = json.loads(env.File(manifest).get_contents())
357 program_dict = obj['program']
358 if env.Bit('pnacl_generate_pexe') and 'portable' in program_dict:
359 return program_dict['portable']['pnacl-translate']['url']
361 return program_dict[env.subst('${TARGET_FULLARCH}')]['url']
364 # Returns scons node for generated manifest.
365 def GeneratedManifestNode(env, manifest):
366 manifest = env.subst(manifest)
367 manifest_base_name = os.path.basename(manifest)
368 main_program = GetMainProgramFromManifest(env, manifest)
369 result = env.File('${STAGING_DIR}/' + manifest_base_name)
370 # Always generate the manifest for nacl_glibc and pnacl pexes.
371 # For nacl_glibc, generating the mapping of shared libraries is non-trivial.
372 # For pnacl, the manifest currently hosts a sha for the pexe.
373 if not env.Bit('nacl_glibc') and not env.Bit('pnacl_generate_pexe'):
374 env.Install('${STAGING_DIR}', manifest)
376 if env.Bit('pnacl_generate_pexe'):
377 return GenerateManifestPnacl(
379 '${STAGING_DIR}/' + manifest_base_name,
381 env.File('${STAGING_DIR}/' + os.path.basename(main_program)))
383 return GenerateManifestDynamicLink(
384 env, '${STAGING_DIR}/' + manifest_base_name,
385 # Note that CopyLibsForExtension() and WhitelistLibsForExtension()
386 # assume that it can find the library list file under this filename.
387 GlibcManifestLibsListFilename(manifest_base_name),
389 env.File('${STAGING_DIR}/' + os.path.basename(main_program)))
393 def GetPnaclExtensionRootNode(env):
394 """Get the scons node representing the root directory of pnacl support files.
396 # This is "built" by src/untrusted/pnacl_support_extension/nacl.scons.
397 return env.Dir('${DESTINATION_ROOT}/pnacl_support')
399 pre_base_env.AddMethod(GetPnaclExtensionRootNode)
401 def GetPnaclExtensionDummyVersion(env):
402 """ We supply a dummy version number when packaging the test-extension
403 that is probably newer than all other versions. """
406 pre_base_env.AddMethod(GetPnaclExtensionDummyVersion)
408 def GetPnaclExtensionNode(env):
409 """Get the scons node representing a specific version of pnacl support files.
411 # This is "built" by src/untrusted/pnacl_support_extension/nacl.scons.
412 return env.Dir('${DESTINATION_ROOT}/pnacl_support/' +
413 env.GetPnaclExtensionDummyVersion())
415 pre_base_env.AddMethod(GetPnaclExtensionNode)
418 # Compares output_file and golden_file.
419 # If they are different, prints the difference and returns 1.
420 # Otherwise, returns 0.
421 def CheckGoldenFile(golden_file, output_file,
422 filter_regex, filter_inverse, filter_group_only):
423 golden = open(golden_file).read()
424 actual = open(output_file).read()
425 if filter_regex is not None:
426 actual = test_lib.RegexpFilterLines(
431 if command_tester.DifferentFromGolden(actual, golden, output_file):
436 # Returns action that compares output_file and golden_file.
437 # This action can be attached to the node with
438 # env.AddPostAction(target, action)
439 def GoldenFileCheckAction(env, output_file, golden_file,
440 filter_regex=None, filter_inverse=False,
441 filter_group_only=False):
442 def ActionFunc(target, source, env):
443 return CheckGoldenFile(env.subst(golden_file), env.subst(output_file),
444 filter_regex, filter_inverse, filter_group_only)
446 return env.Action(ActionFunc)
449 def PPAPIBrowserTester(env,
454 # List of executable basenames to generate
455 # manifest files for.
463 # list of key/value pairs that are passed to the test
465 # list of "--flag=value" pairs (no spaces!)
467 # redirect streams of NaCl program to files
469 nacl_exe_stdout=None,
470 nacl_exe_stderr=None,
471 python_tester_script=None,
473 if 'TRUSTED_ENV' not in env:
476 # No browser tests run on arm-thumb2
477 # Bug http://code.google.com/p/nativeclient/issues/detail?id=2224
478 if env.Bit('target_arm_thumb2'):
481 # Handle issues with mutating any python default arg lists.
482 if browser_flags is None:
485 if env.Bit('pnacl_generate_pexe'):
486 # We likely prefer to choose the 'portable' field in nmfs in this mode.
487 args = args + ['--prefer_portable_in_manifest']
488 # Pass through env var controlling streaming translation
489 if 'NACL_STREAMING_TRANSLATION' in os.environ:
490 env['ENV']['NACL_STREAMING_TRANSLATION'] = 'true'
492 # Lint the extra arguments that are being passed to the tester.
493 special_args = ['--ppapi_plugin', '--sel_ldr', '--irt_library', '--file',
494 '--map_file', '--extension', '--mime_type', '--tool',
495 '--browser_flag', '--test_arg']
496 for arg_name in special_args:
498 raise Exception('%s: %r is a test argument provided by the SCons test'
499 ' wrapper, do not specify it as an additional argument' %
503 env.SetupBrowserEnv()
505 if 'scale_timeout' in ARGUMENTS:
506 timeout = timeout * int(ARGUMENTS['scale_timeout'])
508 if python_tester_script is None:
509 python_tester_script = env.File('${SCONSTRUCT_DIR}/tools/browser_tester'
510 '/browser_tester.py')
511 command = env.GetHeadlessPrefix() + [
512 '${PYTHON}', python_tester_script,
513 '--browser_path', env.ChromeBinary(),
515 # Fail if there is no response for X seconds.
516 '--timeout', str(timeout)]
517 for dep_file in files:
518 command.extend(['--file', dep_file])
519 for extension in extensions:
520 command.extend(['--extension', extension])
521 if env.Bit('bitcode'):
522 # TODO(jvoung): remove this --extension once we have --pnacl-dir working.
523 command.extend(['--extension', env.GetPnaclExtensionNode().abspath])
524 # Enable the installed version of pnacl, and point to a custom install
525 # directory for testing purposes.
526 browser_flags.append('--enable-pnacl')
527 browser_flags.append('--pnacl-dir=%s' %
528 env.GetPnaclExtensionRootNode().abspath)
529 for dest_path, dep_file in map_files:
530 command.extend(['--map_file', dest_path, dep_file])
531 for file_ext, mime_type in mime_types:
532 command.extend(['--mime_type', file_ext, mime_type])
533 command.extend(['--serving_dir', '${NACL_SDK_LIB}'])
534 command.extend(['--serving_dir', '${LIB_DIR}'])
535 if 'browser_tester_bw' in ARGUMENTS:
536 command.extend(['-b', ARGUMENTS['browser_tester_bw']])
538 for nmf_file in nmfs:
539 generated_manifest = GeneratedManifestNode(env, nmf_file)
540 # We need to add generated manifests to the list of default targets.
541 # The manifests should be generated even if the tests are not run -
542 # the manifests may be needed for manual testing.
543 for group in env['COMPONENT_TEST_PROGRAM_GROUPS']:
544 env.Alias(group, generated_manifest)
545 # Generated manifests are served in the root of the HTTP server
546 command.extend(['--file', generated_manifest])
547 for nmf_name in nmf_names:
548 tmp_manifest = '%s.tmp/%s.nmf' % (target, nmf_name)
549 command.extend(['--map_file', '%s.nmf' % nmf_name,
550 env.GenerateSimpleManifest(tmp_manifest, nmf_name)])
551 if 'browser_test_tool' in ARGUMENTS:
552 command.extend(['--tool', ARGUMENTS['browser_test_tool']])
554 # Suppress debugging information on the Chrome waterfall.
555 if env.Bit('disable_flaky_tests') and '--debug' in args:
556 args.remove('--debug')
559 for flag in browser_flags:
560 if flag.find(' ') != -1:
561 raise Exception('Spaces not allowed in browser_flags: '
562 'use --flag=value instead')
563 command.extend(['--browser_flag', flag])
564 for key, value in test_args:
565 command.extend(['--test_arg', str(key), str(value)])
567 # Set a given file to be the nexe's stdin.
568 if nacl_exe_stdin is not None:
569 command.extend(['--nacl_exe_stdin', env.subst(nacl_exe_stdin['file'])])
573 # Set a given file to be the nexe's stdout or stderr. The tester also
574 # compares this output against a golden file.
575 for stream, params in (
576 ('stdout', nacl_exe_stdout),
577 ('stderr', nacl_exe_stderr)):
580 stream_file = env.subst(params['file'])
581 side_effects.append(stream_file)
582 command.extend(['--nacl_exe_' + stream, stream_file])
583 if 'golden' in params:
584 golden_file = env.subst(params['golden'])
585 filter_regex = params.get('filter_regex', None)
586 filter_inverse = params.get('filter_inverse', False)
587 filter_group_only = params.get('filter_group_only', False)
589 GoldenFileCheckAction(
590 env, stream_file, golden_file,
591 filter_regex, filter_inverse, filter_group_only))
593 if env.ShouldUseVerboseOptions(extra):
594 env.MakeVerboseExtraOptions(target, log_verbosity, extra)
595 # Heuristic for when to capture output...
596 capture_output = (extra.pop('capture_output', False)
597 or 'process_output_single' in extra)
598 node = env.CommandTest(target,
600 # Set to 'huge' so that the browser tester's timeout
601 # takes precedence over the default of the test_suite.
603 capture_output=capture_output,
605 # Also indicate that we depend on the layed-out pnacl translator.
606 if env.Bit('bitcode'):
607 env.Depends(node, env.GetPnaclExtensionNode())
608 for side_effect in side_effects:
609 env.SideEffect(side_effect, node)
610 # We can't check output if the test is not run.
611 if not env.Bit('do_not_run_tests'):
612 for action in post_actions:
613 env.AddPostAction(node, action)
616 pre_base_env.AddMethod(PPAPIBrowserTester)
619 # Disabled for ARM and MIPS because Chrome binaries for ARM and MIPS are not
621 def PPAPIBrowserTesterIsBroken(env):
622 return (env.Bit('target_arm') or env.Bit('target_arm_thumb2')
623 or env.Bit('target_mips32'))
625 pre_base_env.AddMethod(PPAPIBrowserTesterIsBroken)
627 # 3D is disabled everywhere
628 def PPAPIGraphics3DIsBroken(env):
631 pre_base_env.AddMethod(PPAPIGraphics3DIsBroken)
634 def AddChromeFilesFromGroup(env, file_group):
635 env['BUILD_SCONSCRIPTS'] = ExtendFileList(
636 env.get('BUILD_SCONSCRIPTS', []),
637 ppapi_scons_files[file_group])
639 pre_base_env.AddMethod(AddChromeFilesFromGroup)