1 # -*- coding: utf-8 -*-
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 from __future__
import absolute_import
, print_function
, unicode_literals
13 from taskgraph
import try_option_syntax
14 from taskgraph
.parameters
import Parameters
15 from taskgraph
.util
.attributes
import match_run_on_projects
, match_run_on_hg_branches
16 from taskgraph
.util
.platforms
import platform_family
18 _target_task_methods
= {}
20 # Some tasks show up in the target task set, but are possibly special cases,
21 # uncommon tasks, or tasks running against limited hardware set that they
22 # should only be selectable with --full.
23 UNCOMMON_TRY_TASK_LABELS
= [
24 # Platforms and/or Build types
25 r
"build-.*-gcp", # Bug 1631990
26 r
"mingwclang", # Bug 1631990
27 r
"valgrind", # Bug 1631990
29 r
"android-geckoview-docs",
32 r
"windows10-64-ref-hw",
35 r
"linux-", # hide all linux32 tasks by default - bug 1599197
36 r
"linux1804-32", # hide linux32 tests - bug 1599197
38 r
"web-platform-tests.*backlog", # hide wpt jobs that are not implemented yet - bug 1572820
40 r
"-profiling-", # talos/raptor profiling jobs are run too often
41 # Hide shippable versions of tests we have opt versions of because the non-shippable
42 # versions are faster to run. This is mostly perf tests.
43 r
"-shippable(?!.*(awsy|browsertime|marionette-headless|mochitest-devtools-chrome-fis|raptor|talos|web-platform-tests-wdspec-headless))", # noqa - too long
47 # These are live site performance tests we run three times a week
68 "stackoverflow-question",
69 "urbandictionary-define",
75 def _target_task(name
):
77 _target_task_methods
[name
] = func
83 def get_method(method
):
84 """Get a target_task_method to pass to a TaskGraphGenerator."""
85 return _target_task_methods
[method
]
88 def filter_out_shipping_phase(task
, parameters
):
90 # nightly still here because of geckodriver
91 not task
.attributes
.get("nightly")
92 and task
.attributes
.get("shipping_phase") in (None, "build")
96 def filter_out_devedition(task
, parameters
):
97 return not task
.attributes
.get("shipping_product") == "devedition"
100 def filter_out_cron(task
, parameters
):
102 Filter out tasks that run via cron.
104 return not task
.attributes
.get("cron")
107 def filter_for_project(task
, parameters
):
108 """Filter tasks by project. Optionally enable nightlies."""
109 run_on_projects
= set(task
.attributes
.get("run_on_projects", []))
110 return match_run_on_projects(parameters
["project"], run_on_projects
)
113 def filter_for_hg_branch(task
, parameters
):
114 """Filter tasks by hg branch.
115 If `run_on_hg_branch` is not defined, then task runs on all branches"""
116 run_on_hg_branches
= set(task
.attributes
.get("run_on_hg_branches", ["all"]))
117 return match_run_on_hg_branches(parameters
["hg_branch"], run_on_hg_branches
)
120 def filter_on_platforms(task
, platforms
):
121 """Filter tasks on the given platform"""
122 platform
= task
.attributes
.get("build_platform")
123 return platform
in platforms
126 def filter_by_uncommon_try_tasks(task
, optional_filters
=None):
127 """Filters tasks that should not be commonly run on try.
130 task (str): String representing the task name.
131 optional_filters (list, optional):
132 Additional filters to apply to task filtering.
135 (Boolean): True if task does not match any known filters.
138 filters
= UNCOMMON_TRY_TASK_LABELS
140 filters
= copy
.deepcopy(filters
)
141 filters
.extend(optional_filters
)
143 return not any(re
.search(pattern
, task
) for pattern
in UNCOMMON_TRY_TASK_LABELS
)
146 def filter_by_regex(task_label
, regexes
, mode
="include"):
147 """Filters tasks according to a list of pre-compiled reguar expressions.
149 If mode is "include", a task label must match any regex to pass.
150 If it is "exclude", a task label must _not_ match any regex to pass.
155 assert mode
in ["include", "exclude"]
157 any_match
= any(r
.search(task_label
) for r
in regexes
)
159 return mode
== "include"
160 return mode
!= "include"
163 def filter_release_tasks(task
, parameters
):
164 platform
= task
.attributes
.get("build_platform")
173 if task
.attributes
["kind"] == "l10n":
174 # This is on-change l10n
177 task
.attributes
["build_type"] == "opt"
178 and task
.attributes
.get("unittest_suite") != "talos"
179 and task
.attributes
.get("unittest_suite") != "raptor"
183 if task
.attributes
.get("shipping_phase") not in (None, "build"):
186 """ No debug on beta/release, keep on ESR with 4 week cycles, beta/release
187 will not be too different from central, but ESR will live for a long time.
189 From June 2019 -> June 2020, we found 1 unique regression on ESR debug
190 and 5 unique regressions on beta/release. Keeping spidermonkey and linux
191 debug finds all but 1 unique regressions (windows found on try) for beta/release.
193 if parameters
["release_type"].startswith("esr"):
196 # code below here is intended to reduce beta/release debug tasks
197 build_type
= task
.attributes
.get("build_type", "")
198 build_platform
= task
.attributes
.get("build_platform", "")
199 test_platform
= task
.attributes
.get("test_platform", "")
200 if task
.kind
== "hazard" or "toolchain" in build_platform
:
201 # keep hazard and toolchain builds around
204 if build_type
== "debug":
205 if "linux" not in build_platform
:
206 # filter out windows/mac/android
208 elif task
.kind
not in ["spidermonkey"] and "-qr" in test_platform
:
209 # filter out linux-qr tests, leave spidermonkey
211 elif "64" not in build_platform
:
212 # filter out linux32 builds
215 # webrender-android-*-debug doesn't have attributes to find 'debug', using task.label.
216 if task
.kind
== "webrender" and "debug" in task
.label
:
221 def filter_out_missing_signoffs(task
, parameters
):
222 for signoff
in parameters
["required_signoffs"]:
223 if signoff
not in parameters
["signoff_urls"] and signoff
in task
.attributes
.get(
224 "required_signoffs", []
230 def filter_tests_without_manifests(task
, parameters
):
231 """Remove test tasks that have an empty 'test_manifests' attribute.
233 This situation can arise when the test loader (e.g bugbug) decided there
234 weren't any important manifests to run for the given push. We filter tasks
235 out here rather than in the transforms so that the full task graph is still
236 aware that the task exists (which is needed by the backfill action).
240 and "test_manifests" in task
.attributes
241 and not task
.attributes
["test_manifests"]
247 def standard_filter(task
, parameters
):
249 filter_func(task
, parameters
)
253 filter_for_hg_branch
,
254 filter_tests_without_manifests
,
259 def accept_raptor_android_build(platform
):
260 """Helper function for selecting the correct android raptor builds."""
261 if "android" not in platform
:
263 if "shippable" not in platform
:
265 if "p2" in platform
and "aarch64" in platform
:
271 def _try_task_config(full_task_graph
, parameters
, graph_config
):
272 requested_tasks
= parameters
["try_task_config"]["tasks"]
273 return list(set(requested_tasks
) & full_task_graph
.graph
.nodes
)
276 def _try_option_syntax(full_task_graph
, parameters
, graph_config
):
277 """Generate a list of target tasks based on try syntax in
278 parameters['message'] and, for context, the full task graph."""
279 options
= try_option_syntax
.TryOptionSyntax(
280 parameters
, full_task_graph
, graph_config
282 target_tasks_labels
= [
284 for t
in six
.itervalues(full_task_graph
.tasks
)
285 if options
.task_matches(t
) and filter_by_uncommon_try_tasks(t
.label
)
289 k
: getattr(options
, k
)
296 for l
in target_tasks_labels
:
297 task
= full_task_graph
[l
]
298 if "unittest_suite" in task
.attributes
:
299 task
.attributes
["task_duplicates"] = options
.trigger_tests
301 for l
in target_tasks_labels
:
302 task
= full_task_graph
[l
]
303 # If the developer wants test jobs to be rebuilt N times we add that value here
304 if options
.trigger_tests
> 1 and "unittest_suite" in task
.attributes
:
305 task
.attributes
["task_duplicates"] = options
.trigger_tests
307 # If the developer wants test talos jobs to be rebuilt N times we add that value here
309 options
.talos_trigger_tests
> 1
310 and task
.attributes
.get("unittest_suite") == "talos"
312 task
.attributes
["task_duplicates"] = options
.talos_trigger_tests
314 # If the developer wants test raptor jobs to be rebuilt N times we add that value here
316 options
.raptor_trigger_tests
317 and options
.raptor_trigger_tests
> 1
318 and task
.attributes
.get("unittest_suite") == "raptor"
320 task
.attributes
["task_duplicates"] = options
.raptor_trigger_tests
322 task
.attributes
.update(attributes
)
324 # Add notifications here as well
325 if options
.notifications
:
326 for task
in full_task_graph
:
327 owner
= parameters
.get("owner")
328 routes
= task
.task
.setdefault("routes", [])
329 if options
.notifications
== "all":
330 routes
.append("notify.email.{}.on-any".format(owner
))
331 elif options
.notifications
== "failure":
332 routes
.append("notify.email.{}.on-failed".format(owner
))
333 routes
.append("notify.email.{}.on-exception".format(owner
))
335 return target_tasks_labels
338 @_target_task("try_tasks")
339 def target_tasks_try(full_task_graph
, parameters
, graph_config
):
340 try_mode
= parameters
["try_mode"]
341 if try_mode
== "try_task_config":
342 return _try_task_config(full_task_graph
, parameters
, graph_config
)
343 elif try_mode
== "try_option_syntax":
344 return _try_option_syntax(full_task_graph
, parameters
, graph_config
)
346 # With no try mode, we schedule nothing, allowing the user to add tasks
347 # later via treeherder.
351 @_target_task("try_select_tasks")
352 def target_tasks_try_select(full_task_graph
, parameters
, graph_config
):
354 for project
in ("autoland", "mozilla-central"):
355 params
= dict(parameters
)
356 params
["project"] = project
357 parameters
= Parameters(**params
)
361 for l
, t
in six
.iteritems(full_task_graph
.tasks
)
362 if standard_filter(t
, parameters
)
363 and filter_out_shipping_phase(t
, parameters
)
364 and filter_out_devedition(t
, parameters
)
368 return [l
for l
in tasks
if filter_by_uncommon_try_tasks(l
)]
371 @_target_task("try_select_tasks_uncommon")
372 def target_tasks_try_select_uncommon(full_task_graph
, parameters
, graph_config
):
374 for project
in ("autoland", "mozilla-central"):
375 params
= dict(parameters
)
376 params
["project"] = project
377 parameters
= Parameters(**params
)
381 for l
, t
in six
.iteritems(full_task_graph
.tasks
)
382 if standard_filter(t
, parameters
)
383 and filter_out_shipping_phase(t
, parameters
)
384 and filter_out_devedition(t
, parameters
)
388 return [l
for l
in tasks
]
391 @_target_task("try_auto")
392 def target_tasks_try_auto(full_task_graph
, parameters
, graph_config
):
393 """Target the tasks which have indicated they should be run on autoland
394 (rather than try) via the `run_on_projects` attributes.
396 Should do the same thing as the `default` target tasks method.
398 params
= dict(parameters
)
399 params
["project"] = "autoland"
400 parameters
= Parameters(**params
)
402 regex_filters
= parameters
["try_task_config"].get("tasks-regex")
403 include_regexes
= exclude_regexes
= []
405 include_regexes
= [re
.compile(r
) for r
in regex_filters
.get("include", [])]
406 exclude_regexes
= [re
.compile(r
) for r
in regex_filters
.get("exclude", [])]
410 for l
, t
in six
.iteritems(full_task_graph
.tasks
)
411 if standard_filter(t
, parameters
)
412 and filter_out_shipping_phase(t
, parameters
)
413 and filter_out_devedition(t
, parameters
)
414 and filter_by_uncommon_try_tasks(t
.label
)
415 and filter_by_regex(t
.label
, include_regexes
, mode
="include")
416 and filter_by_regex(t
.label
, exclude_regexes
, mode
="exclude")
420 @_target_task("default")
421 def target_tasks_default(full_task_graph
, parameters
, graph_config
):
422 """Target the tasks which have indicated they should be run on this project
423 via the `run_on_projects` attributes."""
426 for l
, t
in six
.iteritems(full_task_graph
.tasks
)
427 if standard_filter(t
, parameters
)
428 and filter_out_shipping_phase(t
, parameters
)
429 and filter_out_devedition(t
, parameters
)
433 @_target_task("autoland_tasks")
434 def target_tasks_autoland(full_task_graph
, parameters
, graph_config
):
435 """In addition to doing the filtering by project that the 'default'
436 filter does, also remove any tests running against shippable builds
437 for non-backstop pushes."""
438 filtered_for_project
= target_tasks_default(
439 full_task_graph
, parameters
, graph_config
443 if task
.kind
!= "test":
446 if parameters
["backstop"]:
449 build_type
= task
.attributes
.get("build_type")
450 shippable
= task
.attributes
.get("shippable", False)
452 if not build_type
or build_type
!= "opt" or not shippable
:
457 return [l
for l
in filtered_for_project
if filter(full_task_graph
[l
])]
460 @_target_task("mozilla_central_tasks")
461 def target_tasks_mozilla_central(full_task_graph
, parameters
, graph_config
):
462 """In addition to doing the filtering by project that the 'default'
463 filter does, also remove any tests running against regular (aka not shippable,
464 asan, etc.) opt builds."""
465 filtered_for_project
= target_tasks_default(
466 full_task_graph
, parameters
, graph_config
470 if task
.kind
!= "test":
473 build_platform
= task
.attributes
.get("build_platform")
474 build_type
= task
.attributes
.get("build_type")
475 shippable
= task
.attributes
.get("shippable", False)
477 if not build_platform
or not build_type
:
480 family
= platform_family(build_platform
)
481 # We need to know whether this test is against a "regular" opt build
482 # (which is to say, not shippable, asan, tsan, or any other opt build
483 # with other properties). There's no positive test for this, so we have to
484 # do it somewhat hackily. Android doesn't have variants other than shippable
485 # so it is pretty straightforward to check for. Other platforms have many
486 # variants, but none of the regular opt builds we're looking for have a "-"
487 # in their platform name, so this works (for now).
489 family
== "android" and not shippable
490 ) or "-" not in build_platform
492 if build_type
!= "opt" or not is_regular_opt
:
497 return [l
for l
in filtered_for_project
if filter(full_task_graph
[l
])]
500 @_target_task("graphics_tasks")
501 def target_tasks_graphics(full_task_graph
, parameters
, graph_config
):
502 """In addition to doing the filtering by project that the 'default'
503 filter does, also remove artifact builds because we have csets on
504 the graphics branch that aren't on the candidate branches of artifact
506 filtered_for_project
= target_tasks_default(
507 full_task_graph
, parameters
, graph_config
511 if task
.attributes
["kind"] == "artifact-build":
515 return [l
for l
in filtered_for_project
if filter(full_task_graph
[l
])]
518 @_target_task("mochitest_valgrind")
519 def target_tasks_valgrind(full_task_graph
, parameters
, graph_config
):
520 """Target tasks for mochitest valgrind jobs."""
523 platform
= task
.attributes
.get("test_platform", "").split("/")[0]
524 if platform
not in ["linux64"]:
527 if task
.attributes
.get("unittest_suite", "").startswith(
528 "mochitest-valgrind-plain"
533 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
536 @_target_task("mozilla_beta_tasks")
537 def target_tasks_mozilla_beta(full_task_graph
, parameters
, graph_config
):
538 """Select the set of tasks required for a promotable beta or release build
539 of desktop, plus android CI. The candidates build process involves a pipeline
540 of builds and signing, but does not include beetmover or balrog jobs."""
544 for l
, t
in six
.iteritems(full_task_graph
.tasks
)
545 if filter_release_tasks(t
, parameters
) and standard_filter(t
, parameters
)
549 @_target_task("mozilla_release_tasks")
550 def target_tasks_mozilla_release(full_task_graph
, parameters
, graph_config
):
551 """Select the set of tasks required for a promotable beta or release build
552 of desktop, plus android CI. The candidates build process involves a pipeline
553 of builds and signing, but does not include beetmover or balrog jobs."""
557 for l
, t
in six
.iteritems(full_task_graph
.tasks
)
558 if filter_release_tasks(t
, parameters
) and standard_filter(t
, parameters
)
562 @_target_task("mozilla_esr78_tasks")
563 def target_tasks_mozilla_esr78(full_task_graph
, parameters
, graph_config
):
564 """Select the set of tasks required for a promotable beta or release build
565 of desktop, plus android CI. The candidates build process involves a pipeline
566 of builds and signing, but does not include beetmover or balrog jobs."""
569 if not filter_release_tasks(task
, parameters
):
572 if not standard_filter(task
, parameters
):
575 platform
= task
.attributes
.get("build_platform")
577 # Android is not built on esr78.
578 if platform
and "android" in platform
:
581 test_platform
= task
.attributes
.get("test_platform")
583 # Don't run QuantumRender tests on esr78.
584 if test_platform
and "-qr/" in test_platform
:
589 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
592 @_target_task("promote_desktop")
593 def target_tasks_promote_desktop(full_task_graph
, parameters
, graph_config
):
594 """Select the superset of tasks required to promote a beta or release build
595 of a desktop product. This should include all non-android
596 mozilla_{beta,release} tasks, plus l10n, beetmover, balrog, etc."""
599 if task
.attributes
.get("shipping_product") != parameters
["release_product"]:
602 # 'secondary' balrog/update verify/final verify tasks only run for RCs
603 if parameters
.get("release_type") != "release-rc":
604 if "secondary" in task
.kind
:
607 if not filter_out_missing_signoffs(task
, parameters
):
610 if task
.attributes
.get("shipping_phase") == "promote":
613 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
616 def is_geckoview(task
, parameters
):
618 task
.attributes
.get("shipping_product") == "fennec"
619 and task
.kind
in ("beetmover-geckoview", "upload-symbols")
620 and parameters
["release_product"] == "firefox"
624 @_target_task("push_desktop")
625 def target_tasks_push_desktop(full_task_graph
, parameters
, graph_config
):
626 """Select the set of tasks required to push a build of desktop to cdns.
627 Previous build deps will be optimized out via action task."""
628 filtered_for_candidates
= target_tasks_promote_desktop(
635 if not filter_out_missing_signoffs(task
, parameters
):
637 # Include promotion tasks; these will be optimized out
638 if task
.label
in filtered_for_candidates
:
640 # XXX: Bug 1612540 - include beetmover jobs for publishing geckoview, along
641 # with the regular Firefox (not Devedition!) releases so that they are at sync
642 if "mozilla-esr" not in parameters
["project"] and is_geckoview(
648 task
.attributes
.get("shipping_product") == parameters
["release_product"]
649 and task
.attributes
.get("shipping_phase") == "push"
653 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
656 @_target_task("ship_desktop")
657 def target_tasks_ship_desktop(full_task_graph
, parameters
, graph_config
):
658 """Select the set of tasks required to ship desktop.
659 Previous build deps will be optimized out via action task."""
660 is_rc
= parameters
.get("release_type") == "release-rc"
662 # ship_firefox_rc runs after `promote` rather than `push`; include
664 filtered_for_candidates
= target_tasks_promote_desktop(
670 # ship_firefox runs after `push`; include all push tasks.
671 filtered_for_candidates
= target_tasks_push_desktop(
678 if not filter_out_missing_signoffs(task
, parameters
):
680 # Include promotion tasks; these will be optimized out
681 if task
.label
in filtered_for_candidates
:
684 # XXX: Bug 1619603 - geckoview also ships alongside Firefox RC
685 if is_geckoview(task
, parameters
) and is_rc
:
689 task
.attributes
.get("shipping_product") != parameters
["release_product"]
690 or task
.attributes
.get("shipping_phase") != "ship"
694 if "secondary" in task
.kind
:
699 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
702 @_target_task("pine_tasks")
703 def target_tasks_pine(full_task_graph
, parameters
, graph_config
):
704 """Bug 1339179 - no mobile automation needed on pine"""
707 platform
= task
.attributes
.get("build_platform")
708 # disable mobile jobs
709 if str(platform
).startswith("android"):
712 if platform
== "linux64-asan":
714 # disable non-pine and tasks with a shipping phase
715 if standard_filter(task
, parameters
) or filter_out_shipping_phase(
720 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
723 @_target_task("kaios_tasks")
724 def target_tasks_kaios(full_task_graph
, parameters
, graph_config
):
725 """The set of tasks to run for kaios integration"""
728 # We disable everything in central, and adjust downstream.
731 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
734 @_target_task("ship_geckoview")
735 def target_tasks_ship_geckoview(full_task_graph
, parameters
, graph_config
):
736 """Select the set of tasks required to ship geckoview nightly. The
737 nightly build process involves a pipeline of builds and an upload to
738 maven.mozilla.org."""
741 # XXX Starting 69, we don't ship Fennec Nightly anymore. We just want geckoview to be
743 return task
.attributes
.get("shipping_product") == "fennec" and task
.kind
in (
744 "beetmover-geckoview",
748 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
751 @_target_task("fennec_v68")
752 def target_tasks_fennec_v68(full_task_graph
, parameters
, graph_config
):
754 Select tasks required for running weekly fennec v68 tests
758 test_platform
= task
.attributes
.get("test_platform")
759 try_name
= task
.attributes
.get("raptor_try_name")
761 vismet
= task
.attributes
.get("kind") == "visual-metrics-dep"
763 test_platform
= task
.task
.get("extra").get("treeherder-platform")
764 try_name
= task
.label
766 if task
.attributes
.get("unittest_suite") != "raptor" and not vismet
:
768 if not accept_raptor_android_build(test_platform
):
770 if "-wr" not in try_name
:
773 if "-fennec" in try_name
:
774 if "-power" in try_name
:
776 if "browsertime" in try_name
:
777 if "tp6m" in try_name
:
779 elif "speedometer" in try_name
:
783 if "-youtube-playback" in try_name
:
784 # Bug 1627898: VP9 tests don't work on G5
785 if "-g5-" in test_platform
and "-vp9-" in try_name
:
787 # Bug 1639193: AV1 tests are currently broken
788 if "-av1-" in try_name
:
792 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
795 @_target_task("live_site_perf_testing")
796 def target_tasks_live_site_perf_testing(full_task_graph
, parameters
, graph_config
):
798 Select browsertime live site tasks that should only run once a week.
802 platform
= task
.attributes
.get("build_platform")
803 attributes
= task
.attributes
804 vismet
= attributes
.get("kind") == "visual-metrics-dep"
805 if attributes
.get("unittest_suite") != "raptor" and not vismet
:
807 try_name
= attributes
.get("raptor_try_name")
809 platform
= task
.task
.get("extra").get("treeherder-platform")
810 try_name
= task
.label
812 if not accept_raptor_android_build(platform
):
814 if "-wr" not in try_name
:
816 if "fenix" not in try_name
:
818 if "browsertime" not in try_name
:
820 if "live" not in try_name
:
822 for test
in LIVE_SITES
:
823 if try_name
.endswith(test
+ "-wr") or try_name
.endswith(test
+ "-wr-e10s"):
824 # These tests run 3 times a week, ignore them
829 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
832 @_target_task("general_perf_testing")
833 def target_tasks_general_perf_testing(full_task_graph
, parameters
, graph_config
):
835 Select tasks required for running performance tests 3 times a week.
839 platform
= task
.attributes
.get("test_platform")
840 attributes
= task
.attributes
841 vismet
= attributes
.get("kind") == "visual-metrics-dep"
842 if attributes
.get("unittest_suite") != "raptor" and not vismet
:
845 try_name
= attributes
.get("raptor_try_name")
847 # Visual metric tasks are configured a bit differently
848 platform
= task
.task
.get("extra").get("treeherder-platform")
849 try_name
= task
.label
851 def _run_live_site():
852 for test
in LIVE_SITES
:
853 if try_name
.endswith(test
+ "-wr") or try_name
.endswith(
857 elif try_name
.endswith(test
) or try_name
.endswith(test
+ "-e10s"):
861 # Completely ignore all non-shippable platforms
862 if "shippable" not in platform
:
866 if "android" not in platform
:
867 # Select some browsertime tasks as desktop smoke-tests
868 if "browsertime" in try_name
:
869 if "chrome" in try_name
:
870 if "linux" in platform
or "macos" in platform
:
873 if "chromium" in try_name
:
874 if "linux" in platform
or "macos" in platform
:
877 if "-fis" in try_name
:
879 if "-wr" in try_name
:
881 if "linux" in platform
:
882 if "speedometer" in try_name
:
885 # Run tests on all chrome variants
886 if ("linux" in platform
or "macos" in platform
) and "tp6" in try_name
:
888 if "-chrome" in try_name
:
890 if "-chromium" in try_name
:
893 elif accept_raptor_android_build(platform
):
894 # Ignore all fennec tests here, we run those weekly
895 if "fennec" in try_name
:
897 # Only run webrender tests
898 if "chrome-m" not in try_name
and "-wr" not in try_name
:
900 # Select live site tests
901 if "-live" in try_name
and ("fenix" in try_name
or "chrome-m" in try_name
):
902 return _run_live_site()
903 # Select fenix resource usage tests
904 if "fenix" in try_name
:
905 if "-power" in try_name
:
907 # Select geckoview resource usage tests
908 if "geckoview" in try_name
:
909 # Run cpu+memory, and power tests
910 cpu_n_memory_task
= "-cpu" in try_name
and "-memory" in try_name
911 power_task
= "-power" in try_name
912 # Ignore cpu+memory+power tests
913 if power_task
and cpu_n_memory_task
:
915 if cpu_n_memory_task
:
916 if "-speedometer-" in try_name
:
918 if "-scn" in try_name
and "-idle" in try_name
:
921 return "browsertime" in try_name
922 # Select browsertime-specific tests
923 if "browsertime" in try_name
:
924 if "speedometer" in try_name
:
928 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
931 def make_desktop_nightly_filter(platforms
):
932 """Returns a filter that gets all nightly tasks on the given platform."""
934 def filter(task
, parameters
):
937 filter_on_platforms(task
, platforms
),
938 filter_for_project(task
, parameters
),
939 task
.attributes
.get("shippable", False),
940 # Tests and nightly only builds don't have `shipping_product` set
941 task
.attributes
.get("shipping_product")
942 in {None, "firefox", "thunderbird"},
943 task
.kind
not in {"l10n"}, # no on-change l10n
950 @_target_task("nightly_linux")
951 def target_tasks_nightly_linux(full_task_graph
, parameters
, graph_config
):
952 """Select the set of tasks required for a nightly build of linux. The
953 nightly build process involves a pipeline of builds, signing,
954 and, eventually, uploading the tasks to balrog."""
955 filter = make_desktop_nightly_filter({"linux64-shippable", "linux-shippable"})
956 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
, parameters
)]
959 @_target_task("nightly_macosx")
960 def target_tasks_nightly_macosx(full_task_graph
, parameters
, graph_config
):
961 """Select the set of tasks required for a nightly build of macosx. The
962 nightly build process involves a pipeline of builds, signing,
963 and, eventually, uploading the tasks to balrog."""
964 filter = make_desktop_nightly_filter({"macosx64-shippable"})
965 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
, parameters
)]
968 @_target_task("nightly_win32")
969 def target_tasks_nightly_win32(full_task_graph
, parameters
, graph_config
):
970 """Select the set of tasks required for a nightly build of win32 and win64.
971 The nightly build process involves a pipeline of builds, signing,
972 and, eventually, uploading the tasks to balrog."""
973 filter = make_desktop_nightly_filter({"win32-shippable"})
974 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
, parameters
)]
977 @_target_task("nightly_win64")
978 def target_tasks_nightly_win64(full_task_graph
, parameters
, graph_config
):
979 """Select the set of tasks required for a nightly build of win32 and win64.
980 The nightly build process involves a pipeline of builds, signing,
981 and, eventually, uploading the tasks to balrog."""
982 filter = make_desktop_nightly_filter({"win64-shippable"})
983 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
, parameters
)]
986 @_target_task("nightly_win64_aarch64")
987 def target_tasks_nightly_win64_aarch64(full_task_graph
, parameters
, graph_config
):
988 """Select the set of tasks required for a nightly build of win32 and win64.
989 The nightly build process involves a pipeline of builds, signing,
990 and, eventually, uploading the tasks to balrog."""
991 filter = make_desktop_nightly_filter({"win64-aarch64-shippable"})
992 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
, parameters
)]
995 @_target_task("nightly_asan")
996 def target_tasks_nightly_asan(full_task_graph
, parameters
, graph_config
):
997 """Select the set of tasks required for a nightly build of asan. The
998 nightly build process involves a pipeline of builds, signing,
999 and, eventually, uploading the tasks to balrog."""
1000 filter = make_desktop_nightly_filter(
1001 {"linux64-asan-reporter-shippable", "win64-asan-reporter-shippable"}
1003 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
, parameters
)]
1006 @_target_task("daily_releases")
1007 def target_tasks_daily_releases(full_task_graph
, parameters
, graph_config
):
1008 """Select the set of tasks required to identify if we should release.
1009 If we determine that we should the task will communicate to ship-it to
1010 schedule the release itself."""
1013 return task
.kind
in ["maybe-release"]
1015 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1018 @_target_task("nightly_desktop")
1019 def target_tasks_nightly_desktop(full_task_graph
, parameters
, graph_config
):
1020 """Select the set of tasks required for a nightly build of linux, mac,
1022 # Tasks that aren't platform specific
1023 release_filter
= make_desktop_nightly_filter({None}
)
1026 for l
, t
in six
.iteritems(full_task_graph
.tasks
)
1027 if release_filter(t
, parameters
)
1029 # Avoid duplicate tasks.
1031 set(target_tasks_nightly_win32(full_task_graph
, parameters
, graph_config
))
1032 |
set(target_tasks_nightly_win64(full_task_graph
, parameters
, graph_config
))
1034 target_tasks_nightly_win64_aarch64(
1035 full_task_graph
, parameters
, graph_config
1038 |
set(target_tasks_nightly_macosx(full_task_graph
, parameters
, graph_config
))
1039 |
set(target_tasks_nightly_linux(full_task_graph
, parameters
, graph_config
))
1040 |
set(target_tasks_nightly_asan(full_task_graph
, parameters
, graph_config
))
1041 |
set(release_tasks
)
1045 # Run Searchfox analysis once daily.
1046 @_target_task("searchfox_index")
1047 def target_tasks_searchfox(full_task_graph
, parameters
, graph_config
):
1048 """Select tasks required for indexing Firefox for Searchfox web site each day"""
1050 "searchfox-linux64-searchfox/debug",
1051 "searchfox-macosx64-searchfox/debug",
1052 "searchfox-win64-searchfox/debug",
1053 "searchfox-android-armv7-searchfox/debug",
1054 "source-test-file-metadata-bugzilla-components",
1055 "source-test-file-metadata-test-info-all",
1056 "source-test-wpt-metadata-summary",
1060 # Run Coverity Static Analysis once daily.
1061 @_target_task("coverity_static_analysis_full")
1062 def target_tasks_coverity_full(full_task_graph
, parameters
, graph_config
):
1063 """Select tasks required to run Coverity Static Analysis"""
1064 return ["source-test-coverity-coverity-full-analysis"]
1067 @_target_task("customv8_update")
1068 def target_tasks_customv8_update(full_task_graph
, parameters
, graph_config
):
1069 """Select tasks required for building latest d8/v8 version."""
1070 return ["toolchain-linux64-custom-v8"]
1073 @_target_task("chromium_update")
1074 def target_tasks_chromium_update(full_task_graph
, parameters
, graph_config
):
1075 """Select tasks required for building latest chromium versions."""
1077 "fetch-linux64-chromium",
1078 "fetch-win32-chromium",
1079 "fetch-win64-chromium",
1080 "fetch-mac-chromium",
1084 @_target_task("python_dependency_update")
1085 def target_tasks_python_update(full_task_graph
, parameters
, graph_config
):
1086 """Select the set of tasks required to perform nightly in-tree pipfile updates"""
1089 # For now any task in the repo-update kind is ok
1090 return task
.kind
in ["python-dependency-update"]
1092 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1095 @_target_task("file_update")
1096 def target_tasks_file_update(full_task_graph
, parameters
, graph_config
):
1097 """Select the set of tasks required to perform nightly in-tree file updates"""
1100 # For now any task in the repo-update kind is ok
1101 return task
.kind
in ["repo-update"]
1103 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1106 @_target_task("l10n_bump")
1107 def target_tasks_l10n_bump(full_task_graph
, parameters
, graph_config
):
1108 """Select the set of tasks required to perform l10n bumping."""
1111 # For now any task in the repo-update kind is ok
1112 return task
.kind
in ["l10n-bump"]
1114 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1117 @_target_task("merge_automation")
1118 def target_tasks_merge_automation(full_task_graph
, parameters
, graph_config
):
1119 """Select the set of tasks required to perform repository merges."""
1122 # For now any task in the repo-update kind is ok
1123 return task
.kind
in ["merge-automation"]
1125 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1128 @_target_task("scriptworker_canary")
1129 def target_tasks_scriptworker_canary(full_task_graph
, parameters
, graph_config
):
1130 """Select the set of tasks required to run scriptworker canaries."""
1133 # For now any task in the repo-update kind is ok
1134 return task
.kind
in ["scriptworker-canary"]
1136 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1139 @_target_task("cron_bouncer_check")
1140 def target_tasks_bouncer_check(full_task_graph
, parameters
, graph_config
):
1141 """Select the set of tasks required to perform bouncer version verification."""
1144 if not filter_for_project(task
, parameters
):
1146 # For now any task in the repo-update kind is ok
1147 return task
.kind
in ["cron-bouncer-check"]
1149 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1152 @_target_task("staging_release_builds")
1153 def target_tasks_staging_release(full_task_graph
, parameters
, graph_config
):
1155 Select all builds that are part of releases.
1159 if not task
.attributes
.get("shipping_product"):
1161 if parameters
["release_type"].startswith(
1163 ) and "android" in task
.attributes
.get("build_platform", ""):
1165 if parameters
["release_type"] != "beta" and "devedition" in task
.attributes
.get(
1166 "build_platform", ""
1169 if task
.attributes
.get("shipping_phase") == "build":
1173 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1176 @_target_task("release_simulation")
1177 def target_tasks_release_simulation(full_task_graph
, parameters
, graph_config
):
1179 Select builds that would run on push on a release branch.
1181 project_by_release
= {
1182 "nightly": "mozilla-central",
1183 "beta": "mozilla-beta",
1184 "release": "mozilla-release",
1185 "esr78": "mozilla-esr78",
1187 target_project
= project_by_release
.get(parameters
["release_type"])
1188 if target_project
is None:
1189 raise Exception("Unknown or unspecified release type in simulation run.")
1191 def filter_for_target_project(task
):
1192 """Filter tasks by project. Optionally enable nightlies."""
1193 run_on_projects
= set(task
.attributes
.get("run_on_projects", []))
1194 return match_run_on_projects(target_project
, run_on_projects
)
1196 def filter_out_android_on_esr(task
):
1197 if parameters
["release_type"].startswith(
1199 ) and "android" in task
.attributes
.get("build_platform", ""):
1205 for l
, t
in six
.iteritems(full_task_graph
.tasks
)
1206 if filter_release_tasks(t
, parameters
)
1207 and filter_out_cron(t
, parameters
)
1208 and filter_for_target_project(t
)
1209 and filter_out_android_on_esr(t
)
1213 @_target_task("codereview")
1214 def target_tasks_codereview(full_task_graph
, parameters
, graph_config
):
1215 """Select all code review tasks needed to produce a report"""
1219 if task
.kind
in ["code-review"]:
1223 if task
.attributes
.get("code-review") is True:
1228 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1231 @_target_task("nothing")
1232 def target_tasks_nothing(full_task_graph
, parameters
, graph_config
):
1233 """Select nothing, for DONTBUILD pushes"""
1237 @_target_task("raptor_tp6m")
1238 def target_tasks_raptor_tp6m(full_task_graph
, parameters
, graph_config
):
1240 Select tasks required for running raptor cold page-load tests on fenix and refbrow
1244 platform
= task
.attributes
.get("build_platform")
1245 attributes
= task
.attributes
1247 if platform
and "android" not in platform
:
1249 if attributes
.get("unittest_suite") != "raptor":
1251 try_name
= attributes
.get("raptor_try_name")
1252 if "-cold" in try_name
and "shippable" in platform
:
1253 # Get browsertime amazon smoke tests
1255 "browsertime" in try_name
1256 and "amazon" in try_name
1257 and "search" not in try_name
1258 and "fenix" in try_name
1262 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1265 @_target_task("raptor_tp6_windows10_64_ref_hw_2017")
1266 def target_tasks_raptor_tp6_windows10_64_ref_hw_2017(
1267 full_task_graph
, parameters
, graph_config
1270 Select tasks required for running raptor cold tests on raptor_tp6_windows10_64_ref_hw_2017
1274 platform
= task
.attributes
.get("test_platform")
1275 attributes
= task
.attributes
1277 if attributes
.get("unittest_suite") != "raptor":
1280 if "windows10-64-ref-hw-2017/opt" not in platform
:
1283 try_name
= attributes
.get("raptor_try_name")
1284 if "raptor" in try_name
:
1285 if "-tp6" in try_name
:
1286 if "-cold" in try_name
:
1291 return [l
for l
, t
in six
.iteritems(full_task_graph
.tasks
) if filter(t
)]
1294 @_target_task("condprof")
1295 def target_tasks_condprof(full_task_graph
, parameters
, graph_config
):
1297 Select tasks required for building conditioned profiles.
1299 for name
, task
in six
.iteritems(full_task_graph
.tasks
):
1300 if task
.kind
== "condprof":
1304 @_target_task("system_symbols")
1305 def target_tasks_system_symbols(full_task_graph
, parameters
, graph_config
):
1307 Select tasks for scraping and uploading system symbols.
1309 for name
, task
in six
.iteritems(full_task_graph
.tasks
):
1310 if task
.kind
in ["system-symbols", "system-symbols-upload"]:
1314 @_target_task("perftest")
1315 def target_tasks_perftest(full_task_graph
, parameters
, graph_config
):
1317 Select perftest tasks we want to run daily
1319 for name
, task
in six
.iteritems(full_task_graph
.tasks
):
1320 if task
.kind
!= "perftest":
1322 if task
.attributes
.get("cron", False):
1326 @_target_task("perftest-on-autoland")
1327 def target_tasks_perftest_autoland(full_task_graph
, parameters
, graph_config
):
1329 Select perftest tasks we want to run daily
1331 for name
, task
in six
.iteritems(full_task_graph
.tasks
):
1332 if task
.kind
!= "perftest":
1334 if task
.attributes
.get("cron", False) and any(
1335 test_name
in name
for test_name
in ["view"]