1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 from datetime
import datetime
, timedelta
11 from redo
import retry
12 from taskgraph
.parameters
import Parameters
13 from taskgraph
.target_tasks
import _target_task
, get_method
14 from taskgraph
.util
.taskcluster
import find_task_id
16 from gecko_taskgraph
import GECKO
, try_option_syntax
17 from gecko_taskgraph
.util
.attributes
import (
18 match_run_on_hg_branches
,
19 match_run_on_projects
,
21 from gecko_taskgraph
.util
.hg
import find_hg_revision_push_info
, get_hg_commit_message
22 from gecko_taskgraph
.util
.platforms
import platform_family
24 # Some tasks show up in the target task set, but are possibly special cases,
25 # uncommon tasks, or tasks running against limited hardware set that they
26 # should only be selectable with --full.
27 UNCOMMON_TRY_TASK_LABELS
= [
28 # Platforms and/or Build types
29 r
"build-.*-gcp", # Bug 1631990
30 r
"mingwclang", # Bug 1631990
31 r
"valgrind", # Bug 1631990
33 r
"android-geckoview-docs",
36 r
"windows10-64-ref-hw",
37 r
"windows10-aarch64-qr",
39 r
"linux-", # hide all linux32 tasks by default - bug 1599197
40 r
"linux1804-32", # hide linux32 tests - bug 1599197
42 r
"web-platform-tests.*backlog", # hide wpt jobs that are not implemented yet - bug 1572820
44 r
"-profiling-", # talos/raptor profiling jobs are run too often
45 r
"-32-.*-webgpu", # webgpu gets little benefit from these tests.
48 # Hide shippable versions of tests we have opt versions of because the non-shippable
49 # versions are faster to run. This is mostly perf tests.
50 r
"-shippable(?!.*(awsy|browsertime|marionette-headless|mochitest-devtools-chrome-fis|raptor|talos|web-platform-tests-wdspec-headless|mochitest-plain-headless))", # noqa - too long
54 def index_exists(index_path
, reason
=""):
55 print(f
"Looking for existing index {index_path} {reason}...")
57 task_id
= find_task_id(index_path
)
58 print(f
"Index {index_path} exists: taskId {task_id}")
61 print(f
"Index {index_path} doesn't exist.")
65 def filter_out_shipping_phase(task
, parameters
):
67 # nightly still here because of geckodriver
68 not task
.attributes
.get("nightly")
69 and task
.attributes
.get("shipping_phase") in (None, "build")
73 def filter_out_devedition(task
, parameters
):
74 return not task
.attributes
.get("shipping_product") == "devedition"
77 def filter_out_cron(task
, parameters
):
79 Filter out tasks that run via cron.
81 return not task
.attributes
.get("cron")
84 def filter_for_project(task
, parameters
):
85 """Filter tasks by project. Optionally enable nightlies."""
86 run_on_projects
= set(task
.attributes
.get("run_on_projects", []))
87 return match_run_on_projects(parameters
["project"], run_on_projects
)
90 def filter_for_hg_branch(task
, parameters
):
91 """Filter tasks by hg branch.
92 If `run_on_hg_branch` is not defined, then task runs on all branches"""
93 run_on_hg_branches
= set(task
.attributes
.get("run_on_hg_branches", ["all"]))
94 return match_run_on_hg_branches(parameters
["hg_branch"], run_on_hg_branches
)
97 def filter_on_platforms(task
, platforms
):
98 """Filter tasks on the given platform"""
99 platform
= task
.attributes
.get("build_platform")
100 return platform
in platforms
103 def filter_by_uncommon_try_tasks(task
, optional_filters
=None):
104 """Filters tasks that should not be commonly run on try.
107 task (str): String representing the task name.
108 optional_filters (list, optional):
109 Additional filters to apply to task filtering.
112 (Boolean): True if task does not match any known filters.
115 filters
= UNCOMMON_TRY_TASK_LABELS
117 filters
= itertools
.chain(filters
, optional_filters
)
119 return not any(re
.search(pattern
, task
) for pattern
in filters
)
122 def filter_by_regex(task_label
, regexes
, mode
="include"):
123 """Filters tasks according to a list of pre-compiled reguar expressions.
125 If mode is "include", a task label must match any regex to pass.
126 If it is "exclude", a task label must _not_ match any regex to pass.
131 assert mode
in ["include", "exclude"]
133 any_match
= any(r
.search(task_label
) for r
in regexes
)
135 return mode
== "include"
136 return mode
!= "include"
139 def filter_release_tasks(task
, parameters
):
140 platform
= task
.attributes
.get("build_platform")
149 if task
.attributes
["kind"] == "l10n":
150 # This is on-change l10n
153 task
.attributes
["build_type"] == "opt"
154 and task
.attributes
.get("unittest_suite") != "talos"
155 and task
.attributes
.get("unittest_suite") != "raptor"
159 if task
.attributes
.get("shipping_phase") not in (None, "build"):
162 """ No debug on release, keep on ESR with 4 week cycles, release
163 will not be too different from central, but ESR will live for a long time.
165 From June 2019 -> June 2020, we found 1 unique regression on ESR debug
166 and 5 unique regressions on beta/release. Keeping spidermonkey and linux
167 debug finds all but 1 unique regressions (windows found on try) for beta/release.
169 ...but debug-only failures started showing up on ESR (esr-91, esr-102) so
170 desktop debug tests were added back for beta.
172 build_type
= task
.attributes
.get("build_type", "")
173 build_platform
= task
.attributes
.get("build_platform", "")
174 test_platform
= task
.attributes
.get("test_platform", "")
176 if parameters
["release_type"].startswith("esr") or (
177 parameters
["release_type"] == "beta" and "android" not in build_platform
181 # code below here is intended to reduce release debug tasks
182 if task
.kind
== "hazard" or "toolchain" in build_platform
:
183 # keep hazard and toolchain builds around
186 if build_type
== "debug":
187 if "linux" not in build_platform
:
188 # filter out windows/mac/android
190 if task
.kind
not in ["spidermonkey"] and "-qr" in test_platform
:
191 # filter out linux-qr tests, leave spidermonkey
193 if "64" not in build_platform
:
194 # filter out linux32 builds
197 # webrender-android-*-debug doesn't have attributes to find 'debug', using task.label.
198 if task
.kind
== "webrender" and "debug" in task
.label
:
203 def filter_out_missing_signoffs(task
, parameters
):
204 for signoff
in parameters
["required_signoffs"]:
205 if signoff
not in parameters
["signoff_urls"] and signoff
in task
.attributes
.get(
206 "required_signoffs", []
212 def filter_tests_without_manifests(task
, parameters
):
213 """Remove test tasks that have an empty 'test_manifests' attribute.
215 This situation can arise when the test loader (e.g bugbug) decided there
216 weren't any important manifests to run for the given push. We filter tasks
217 out here rather than in the transforms so that the full task graph is still
218 aware that the task exists (which is needed by the backfill action).
222 and "test_manifests" in task
.attributes
223 and not task
.attributes
["test_manifests"]
229 def standard_filter(task
, parameters
):
231 filter_func(task
, parameters
)
235 filter_for_hg_branch
,
236 filter_tests_without_manifests
,
241 def accept_raptor_android_build(platform
):
242 """Helper function for selecting the correct android raptor builds."""
243 if "android" not in platform
:
245 if "shippable" not in platform
:
247 if "p2" in platform
and "aarch64" in platform
:
249 if "p5" in platform
and "aarch64" in platform
:
253 if "a51" in platform
:
257 def filter_unsupported_artifact_builds(task
, parameters
):
258 try_config
= parameters
.get("try_task_config", {})
259 if not try_config
.get("use-artifact-builds", False):
262 supports_artifact_builds
= task
.attributes
.get("supports-artifact-builds", True)
263 return supports_artifact_builds
266 def filter_out_shippable(task
):
267 return not task
.attributes
.get("shippable", False)
270 def _try_task_config(full_task_graph
, parameters
, graph_config
):
271 requested_tasks
= parameters
["try_task_config"]["tasks"]
272 return list(set(requested_tasks
) & full_task_graph
.graph
.nodes
)
275 def _try_option_syntax(full_task_graph
, parameters
, graph_config
):
276 """Generate a list of target tasks based on try syntax in
277 parameters['message'] and, for context, the full task graph."""
278 options
= try_option_syntax
.TryOptionSyntax(
279 parameters
, full_task_graph
, graph_config
281 target_tasks_labels
= [
283 for t
in full_task_graph
.tasks
.values()
284 if options
.task_matches(t
)
285 and filter_by_uncommon_try_tasks(t
.label
)
286 and filter_unsupported_artifact_builds(t
, parameters
)
290 k
: getattr(options
, k
)
297 for l
in target_tasks_labels
:
298 task
= full_task_graph
[l
]
299 if "unittest_suite" in task
.attributes
:
300 task
.attributes
["task_duplicates"] = options
.trigger_tests
302 for l
in target_tasks_labels
:
303 task
= full_task_graph
[l
]
304 # If the developer wants test jobs to be rebuilt N times we add that value here
305 if options
.trigger_tests
> 1 and "unittest_suite" in task
.attributes
:
306 task
.attributes
["task_duplicates"] = options
.trigger_tests
308 # If the developer wants test talos jobs to be rebuilt N times we add that value here
310 options
.talos_trigger_tests
> 1
311 and task
.attributes
.get("unittest_suite") == "talos"
313 task
.attributes
["task_duplicates"] = options
.talos_trigger_tests
315 # If the developer wants test raptor jobs to be rebuilt N times we add that value here
317 options
.raptor_trigger_tests
318 and options
.raptor_trigger_tests
> 1
319 and task
.attributes
.get("unittest_suite") == "raptor"
321 task
.attributes
["task_duplicates"] = options
.raptor_trigger_tests
323 task
.attributes
.update(attributes
)
325 # Add notifications here as well
326 if options
.notifications
:
327 for task
in full_task_graph
:
328 owner
= parameters
.get("owner")
329 routes
= task
.task
.setdefault("routes", [])
330 if options
.notifications
== "all":
331 routes
.append(f
"notify.email.{owner}.on-any")
332 elif options
.notifications
== "failure":
333 routes
.append(f
"notify.email.{owner}.on-failed")
334 routes
.append(f
"notify.email.{owner}.on-exception")
336 return target_tasks_labels
339 @_target_task("try_tasks")
340 def target_tasks_try(full_task_graph
, parameters
, graph_config
):
341 try_mode
= parameters
["try_mode"]
342 if try_mode
== "try_task_config":
343 return _try_task_config(full_task_graph
, parameters
, graph_config
)
344 if try_mode
== "try_option_syntax":
345 return _try_option_syntax(full_task_graph
, parameters
, graph_config
)
346 # With no try mode, we schedule nothing, allowing the user to add tasks
347 # later via treeherder.
351 @_target_task("try_select_tasks")
352 def target_tasks_try_select(full_task_graph
, parameters
, graph_config
):
353 tasks
= target_tasks_try_select_uncommon(full_task_graph
, parameters
, graph_config
)
354 return [l
for l
in tasks
if filter_by_uncommon_try_tasks(l
)]
357 @_target_task("try_select_tasks_uncommon")
358 def target_tasks_try_select_uncommon(full_task_graph
, parameters
, graph_config
):
359 from gecko_taskgraph
.decision
import PER_PROJECT_PARAMETERS
361 projects
= ("autoland", "mozilla-central")
362 if parameters
["project"] not in projects
:
363 projects
= (parameters
["project"],)
366 for project
in projects
:
367 params
= dict(parameters
)
368 params
["project"] = project
369 parameters
= Parameters(**params
)
372 target_tasks_method
= PER_PROJECT_PARAMETERS
[project
]["target_tasks_method"]
374 target_tasks_method
= "default"
377 get_method(target_tasks_method
)(full_task_graph
, parameters
, graph_config
)
383 @_target_task("try_auto")
384 def target_tasks_try_auto(full_task_graph
, parameters
, graph_config
):
385 """Target the tasks which have indicated they should be run on autoland
386 (rather than try) via the `run_on_projects` attributes.
388 Should do the same thing as the `default` target tasks method.
390 params
= dict(parameters
)
391 params
["project"] = "autoland"
392 parameters
= Parameters(**params
)
394 regex_filters
= parameters
["try_task_config"].get("tasks-regex")
395 include_regexes
= exclude_regexes
= []
397 include_regexes
= [re
.compile(r
) for r
in regex_filters
.get("include", [])]
398 exclude_regexes
= [re
.compile(r
) for r
in regex_filters
.get("exclude", [])]
402 for l
, t
in full_task_graph
.tasks
.items()
403 if standard_filter(t
, parameters
)
404 and filter_out_shipping_phase(t
, parameters
)
405 and filter_out_devedition(t
, parameters
)
406 and filter_by_uncommon_try_tasks(t
.label
)
407 and filter_by_regex(t
.label
, include_regexes
, mode
="include")
408 and filter_by_regex(t
.label
, exclude_regexes
, mode
="exclude")
409 and filter_unsupported_artifact_builds(t
, parameters
)
410 and filter_out_shippable(t
)
414 @_target_task("default")
415 def target_tasks_default(full_task_graph
, parameters
, graph_config
):
416 """Target the tasks which have indicated they should be run on this project
417 via the `run_on_projects` attributes."""
420 for l
, t
in full_task_graph
.tasks
.items()
421 if standard_filter(t
, parameters
)
422 and filter_out_shipping_phase(t
, parameters
)
423 and filter_out_devedition(t
, parameters
)
427 @_target_task("autoland_tasks")
428 def target_tasks_autoland(full_task_graph
, parameters
, graph_config
):
429 """In addition to doing the filtering by project that the 'default'
430 filter does, also remove any tests running against shippable builds
431 for non-backstop pushes."""
432 filtered_for_project
= target_tasks_default(
433 full_task_graph
, parameters
, graph_config
437 if task
.kind
!= "test":
440 if parameters
["backstop"]:
443 build_type
= task
.attributes
.get("build_type")
445 if not build_type
or build_type
!= "opt" or filter_out_shippable(task
):
450 return [l
for l
in filtered_for_project
if filter(full_task_graph
[l
])]
453 @_target_task("mozilla_central_tasks")
454 def target_tasks_mozilla_central(full_task_graph
, parameters
, graph_config
):
455 """In addition to doing the filtering by project that the 'default'
456 filter does, also remove any tests running against regular (aka not shippable,
457 asan, etc.) opt builds."""
458 filtered_for_project
= target_tasks_default(
459 full_task_graph
, parameters
, graph_config
463 if task
.kind
!= "test":
466 build_platform
= task
.attributes
.get("build_platform")
467 build_type
= task
.attributes
.get("build_type")
468 shippable
= task
.attributes
.get("shippable", False)
470 if not build_platform
or not build_type
:
473 family
= platform_family(build_platform
)
474 # We need to know whether this test is against a "regular" opt build
475 # (which is to say, not shippable, asan, tsan, or any other opt build
476 # with other properties). There's no positive test for this, so we have to
477 # do it somewhat hackily. Android doesn't have variants other than shippable
478 # so it is pretty straightforward to check for. Other platforms have many
479 # variants, but none of the regular opt builds we're looking for have a "-"
480 # in their platform name, so this works (for now).
482 family
== "android" and not shippable
483 ) or "-" not in build_platform
485 if build_type
!= "opt" or not is_regular_opt
:
490 return [l
for l
in filtered_for_project
if filter(full_task_graph
[l
])]
493 @_target_task("graphics_tasks")
494 def target_tasks_graphics(full_task_graph
, parameters
, graph_config
):
495 """In addition to doing the filtering by project that the 'default'
496 filter does, also remove artifact builds because we have csets on
497 the graphics branch that aren't on the candidate branches of artifact
499 filtered_for_project
= target_tasks_default(
500 full_task_graph
, parameters
, graph_config
504 if task
.attributes
["kind"] == "artifact-build":
508 return [l
for l
in filtered_for_project
if filter(full_task_graph
[l
])]
511 @_target_task("mozilla_beta_tasks")
512 def target_tasks_mozilla_beta(full_task_graph
, parameters
, graph_config
):
513 """Select the set of tasks required for a promotable beta or release build
514 of desktop, plus android CI. The candidates build process involves a pipeline
515 of builds and signing, but does not include beetmover or balrog jobs."""
519 for l
, t
in full_task_graph
.tasks
.items()
520 if filter_release_tasks(t
, parameters
) and standard_filter(t
, parameters
)
524 @_target_task("mozilla_release_tasks")
525 def target_tasks_mozilla_release(full_task_graph
, parameters
, graph_config
):
526 """Select the set of tasks required for a promotable beta or release build
527 of desktop, plus android CI. The candidates build process involves a pipeline
528 of builds and signing, but does not include beetmover or balrog jobs."""
532 for l
, t
in full_task_graph
.tasks
.items()
533 if filter_release_tasks(t
, parameters
) and standard_filter(t
, parameters
)
537 @_target_task("mozilla_esr102_tasks")
538 def target_tasks_mozilla_esr102(full_task_graph
, parameters
, graph_config
):
539 """Select the set of tasks required for a promotable beta or release build
540 of desktop, without android CI. The candidates build process involves a pipeline
541 of builds and signing, but does not include beetmover or balrog jobs."""
544 if not filter_release_tasks(task
, parameters
):
547 if not standard_filter(task
, parameters
):
550 platform
= task
.attributes
.get("build_platform")
552 # Android is not built on esr102.
553 if platform
and "android" in platform
:
558 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
561 @_target_task("mozilla_esr115_tasks")
562 def target_tasks_mozilla_esr115(full_task_graph
, parameters
, graph_config
):
563 """Select the set of tasks required for a promotable beta or release build
564 of desktop, without android CI. The candidates build process involves a pipeline
565 of builds and signing, but does not include beetmover or balrog jobs."""
568 if not filter_release_tasks(task
, parameters
):
571 if not standard_filter(task
, parameters
):
574 platform
= task
.attributes
.get("build_platform")
576 # Android is not built on esr115.
577 if platform
and "android" in platform
:
582 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
585 @_target_task("promote_desktop")
586 def target_tasks_promote_desktop(full_task_graph
, parameters
, graph_config
):
587 """Select the superset of tasks required to promote a beta or release build
588 of a desktop product. This should include all non-android
589 mozilla_{beta,release} tasks, plus l10n, beetmover, balrog, etc."""
592 # Bug 1758507 - geckoview ships in the promote phase
593 if not parameters
["release_type"].startswith("esr") and is_geckoview(
598 if task
.attributes
.get("shipping_product") != parameters
["release_product"]:
601 # 'secondary' balrog/update verify/final verify tasks only run for RCs
602 if parameters
.get("release_type") != "release-rc":
603 if "secondary" in task
.kind
:
606 if not filter_out_missing_signoffs(task
, parameters
):
609 if task
.attributes
.get("shipping_phase") == "promote":
612 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
615 def is_geckoview(task
, parameters
):
617 task
.attributes
.get("shipping_product") == "fennec"
618 and task
.kind
in ("beetmover-geckoview", "upload-symbols")
619 and parameters
["release_product"] == "firefox"
623 @_target_task("push_desktop")
624 def target_tasks_push_desktop(full_task_graph
, parameters
, graph_config
):
625 """Select the set of tasks required to push a build of desktop to cdns.
626 Previous build deps will be optimized out via action task."""
627 filtered_for_candidates
= target_tasks_promote_desktop(
634 if not filter_out_missing_signoffs(task
, parameters
):
636 # Include promotion tasks; these will be optimized out
637 if task
.label
in filtered_for_candidates
:
641 task
.attributes
.get("shipping_product") == parameters
["release_product"]
642 and task
.attributes
.get("shipping_phase") == "push"
646 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
649 @_target_task("ship_desktop")
650 def target_tasks_ship_desktop(full_task_graph
, parameters
, graph_config
):
651 """Select the set of tasks required to ship desktop.
652 Previous build deps will be optimized out via action task."""
653 is_rc
= parameters
.get("release_type") == "release-rc"
655 # ship_firefox_rc runs after `promote` rather than `push`; include
657 filtered_for_candidates
= target_tasks_promote_desktop(
663 # ship_firefox runs after `push`; include all push tasks.
664 filtered_for_candidates
= target_tasks_push_desktop(
671 if not filter_out_missing_signoffs(task
, parameters
):
673 # Include promotion tasks; these will be optimized out
674 if task
.label
in filtered_for_candidates
:
678 task
.attributes
.get("shipping_product") != parameters
["release_product"]
679 or task
.attributes
.get("shipping_phase") != "ship"
683 if "secondary" in task
.kind
:
687 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
690 @_target_task("pine_tasks")
691 def target_tasks_pine(full_task_graph
, parameters
, graph_config
):
692 """Bug 1339179 - no mobile automation needed on pine"""
695 platform
= task
.attributes
.get("build_platform")
696 # disable mobile jobs
697 if str(platform
).startswith("android"):
700 if platform
== "linux64-asan":
702 # disable non-pine and tasks with a shipping phase
703 if standard_filter(task
, parameters
) or filter_out_shipping_phase(
708 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
711 @_target_task("kaios_tasks")
712 def target_tasks_kaios(full_task_graph
, parameters
, graph_config
):
713 """The set of tasks to run for kaios integration"""
716 # We disable everything in central, and adjust downstream.
719 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
722 @_target_task("ship_geckoview")
723 def target_tasks_ship_geckoview(full_task_graph
, parameters
, graph_config
):
724 """Select the set of tasks required to ship geckoview nightly. The
725 nightly build process involves a pipeline of builds and an upload to
726 maven.mozilla.org."""
728 f
"{graph_config['trust-domain']}.v2.{parameters['project']}.revision."
729 f
"{parameters['head_rev']}.taskgraph.decision-ship-geckoview"
731 if os
.environ
.get("MOZ_AUTOMATION") and retry(
735 "reason": "to avoid triggering multiple nightlies off the same revision",
741 # XXX Starting 69, we don't ship Fennec Nightly anymore. We just want geckoview to be
743 return task
.attributes
.get("shipping_product") == "fennec" and task
.kind
in (
744 "beetmover-geckoview",
748 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
751 @_target_task("custom-car_perf_testing")
752 def target_tasks_custom_car_perf_testing(full_task_graph
, parameters
, graph_config
):
753 """Select tasks required for running daily performance tests for custom chromium-as-release."""
756 platform
= task
.attributes
.get("test_platform")
757 attributes
= task
.attributes
758 if attributes
.get("unittest_suite") != "raptor":
761 try_name
= attributes
.get("raptor_try_name")
763 # Completely ignore all non-shippable platforms
764 if "shippable" not in platform
:
767 # ignore all windows 7 perf jobs scheduled automatically
768 if "windows10-32" in platform
:
771 # Desktop selection only for CaR
772 if "android" not in platform
:
773 if "browsertime" in try_name
and "custom-car" in try_name
:
777 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
780 @_target_task("general_perf_testing")
781 def target_tasks_general_perf_testing(full_task_graph
, parameters
, graph_config
):
783 Select tasks required for running performance tests 3 times a week.
787 platform
= task
.attributes
.get("test_platform")
788 attributes
= task
.attributes
789 if attributes
.get("unittest_suite") != "raptor":
792 try_name
= attributes
.get("raptor_try_name")
794 # Completely ignore all non-shippable platforms
795 if "shippable" not in platform
:
798 # ignore all windows 7 perf jobs scheduled automatically
799 if "windows7" in platform
or "windows10-32" in platform
:
803 if "android" not in platform
:
804 # Select some browsertime tasks as desktop smoke-tests
805 if "browsertime" in try_name
:
806 if "chrome" in try_name
:
808 if "chromium" in try_name
:
810 # chromium-as-release has it's own cron
811 if "custom-car" in try_name
:
813 if "-live" in try_name
:
815 if "-fis" in try_name
:
817 if "linux" in platform
:
818 if "speedometer" in try_name
:
820 if "safari" and "benchmark" in try_name
:
821 # Speedometer 3 is broken on Safari, see bug 1802922
822 if "speedometer3" in try_name
:
826 # Don't run tp6 raptor tests
827 if "tp6" in try_name
:
830 elif accept_raptor_android_build(platform
):
831 if "chrome-m" in try_name
and (
832 ("ebay" in try_name
and "live" not in try_name
)
835 and ("facebook" in try_name
or "dailymail" in try_name
)
839 # Ignore all fennec tests here, we run those weekly
840 if "fennec" in try_name
:
842 # Only run webrender tests
843 if "chrome-m" not in try_name
and "-qr" not in platform
:
845 # Select live site tests
846 if "-live" in try_name
:
848 # Select fenix resource usage tests
849 if "fenix" in try_name
:
850 # Bug 1816421 disable fission perf tests
851 if "-fis" in try_name
:
853 if "-power" in try_name
:
855 # Select geckoview resource usage tests
856 if "geckoview" in try_name
:
857 # Bug 1816421 disable fission perf tests
858 if "-fis" in try_name
:
860 # Run cpu+memory, and power tests
861 cpu_n_memory_task
= "-cpu" in try_name
and "-memory" in try_name
862 power_task
= "-power" in try_name
863 # Ignore cpu+memory+power tests
864 if power_task
and cpu_n_memory_task
:
866 if cpu_n_memory_task
:
869 return "browsertime" in try_name
870 # Select browsertime-specific tests
871 if "browsertime" in try_name
:
872 if "speedometer" in try_name
:
876 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
879 def make_desktop_nightly_filter(platforms
):
880 """Returns a filter that gets all nightly tasks on the given platform."""
882 def filter(task
, parameters
):
885 filter_on_platforms(task
, platforms
),
886 filter_for_project(task
, parameters
),
887 task
.attributes
.get("shippable", False),
888 # Tests and nightly only builds don't have `shipping_product` set
889 task
.attributes
.get("shipping_product")
890 in {None, "firefox", "thunderbird"},
891 task
.kind
not in {"l10n"}, # no on-change l10n
898 @_target_task("sp-perftests")
899 def target_tasks_speedometer_tests(full_task_graph
, parameters
, graph_config
):
901 platform
= task
.attributes
.get("test_platform")
902 attributes
= task
.attributes
903 if attributes
.get("unittest_suite") != "raptor":
905 if "windows10-32" not in platform
:
906 try_name
= attributes
.get("raptor_try_name")
908 "browsertime" in try_name
909 and "speedometer" in try_name
910 and "chrome" in try_name
914 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
917 @_target_task("nightly_linux")
918 def target_tasks_nightly_linux(full_task_graph
, parameters
, graph_config
):
919 """Select the set of tasks required for a nightly build of linux. The
920 nightly build process involves a pipeline of builds, signing,
921 and, eventually, uploading the tasks to balrog."""
922 filter = make_desktop_nightly_filter({"linux64-shippable", "linux-shippable"})
923 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
926 @_target_task("nightly_macosx")
927 def target_tasks_nightly_macosx(full_task_graph
, parameters
, graph_config
):
928 """Select the set of tasks required for a nightly build of macosx. The
929 nightly build process involves a pipeline of builds, signing,
930 and, eventually, uploading the tasks to balrog."""
931 filter = make_desktop_nightly_filter({"macosx64-shippable"})
932 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
935 @_target_task("nightly_win32")
936 def target_tasks_nightly_win32(full_task_graph
, parameters
, graph_config
):
937 """Select the set of tasks required for a nightly build of win32 and win64.
938 The nightly build process involves a pipeline of builds, signing,
939 and, eventually, uploading the tasks to balrog."""
940 filter = make_desktop_nightly_filter({"win32-shippable"})
941 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
944 @_target_task("nightly_win64")
945 def target_tasks_nightly_win64(full_task_graph
, parameters
, graph_config
):
946 """Select the set of tasks required for a nightly build of win32 and win64.
947 The nightly build process involves a pipeline of builds, signing,
948 and, eventually, uploading the tasks to balrog."""
949 filter = make_desktop_nightly_filter({"win64-shippable"})
950 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
953 @_target_task("nightly_win64_aarch64")
954 def target_tasks_nightly_win64_aarch64(full_task_graph
, parameters
, graph_config
):
955 """Select the set of tasks required for a nightly build of win32 and win64.
956 The nightly build process involves a pipeline of builds, signing,
957 and, eventually, uploading the tasks to balrog."""
958 filter = make_desktop_nightly_filter({"win64-aarch64-shippable"})
959 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
962 @_target_task("nightly_asan")
963 def target_tasks_nightly_asan(full_task_graph
, parameters
, graph_config
):
964 """Select the set of tasks required for a nightly build of asan. The
965 nightly build process involves a pipeline of builds, signing,
966 and, eventually, uploading the tasks to balrog."""
967 filter = make_desktop_nightly_filter(
968 {"linux64-asan-reporter-shippable", "win64-asan-reporter-shippable"}
970 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
973 @_target_task("daily_releases")
974 def target_tasks_daily_releases(full_task_graph
, parameters
, graph_config
):
975 """Select the set of tasks required to identify if we should release.
976 If we determine that we should the task will communicate to ship-it to
977 schedule the release itself."""
980 return task
.kind
in ["maybe-release"]
982 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
985 @_target_task("nightly_desktop")
986 def target_tasks_nightly_desktop(full_task_graph
, parameters
, graph_config
):
987 """Select the set of tasks required for a nightly build of linux, mac,
990 f
"{graph_config['trust-domain']}.v2.{parameters['project']}.revision."
991 f
"{parameters['head_rev']}.taskgraph.decision-nightly-desktop"
993 if os
.environ
.get("MOZ_AUTOMATION") and retry(
997 "reason": "to avoid triggering multiple nightlies off the same revision",
1002 # Tasks that aren't platform specific
1003 release_filter
= make_desktop_nightly_filter({None}
)
1005 l
for l
, t
in full_task_graph
.tasks
.items() if release_filter(t
, parameters
)
1007 # Avoid duplicate tasks.
1009 set(target_tasks_nightly_win32(full_task_graph
, parameters
, graph_config
))
1010 |
set(target_tasks_nightly_win64(full_task_graph
, parameters
, graph_config
))
1012 target_tasks_nightly_win64_aarch64(
1013 full_task_graph
, parameters
, graph_config
1016 |
set(target_tasks_nightly_macosx(full_task_graph
, parameters
, graph_config
))
1017 |
set(target_tasks_nightly_linux(full_task_graph
, parameters
, graph_config
))
1018 |
set(target_tasks_nightly_asan(full_task_graph
, parameters
, graph_config
))
1019 |
set(release_tasks
)
1023 # Run Searchfox analysis once daily.
1024 @_target_task("searchfox_index")
1025 def target_tasks_searchfox(full_task_graph
, parameters
, graph_config
):
1026 """Select tasks required for indexing Firefox for Searchfox web site each day"""
1028 "searchfox-linux64-searchfox/debug",
1029 "searchfox-macosx64-searchfox/debug",
1030 "searchfox-win64-searchfox/debug",
1031 "searchfox-android-armv7-searchfox/debug",
1032 "source-test-file-metadata-bugzilla-components",
1033 "source-test-file-metadata-test-info-all",
1034 "source-test-wpt-metadata-summary",
1038 # Run build linux64-plain-clang-trunk/opt on mozilla-central/beta with perf tests
1039 @_target_task("linux64_clang_trunk_perf")
1040 def target_tasks_build_linux64_clang_trunk_perf(
1041 full_task_graph
, parameters
, graph_config
1043 """Select tasks required to run perf test on linux64 build with clang trunk"""
1045 # Only keep tasks generated from platform `linux1804-64-clang-trunk-qr/opt`
1046 def filter(task_label
):
1047 if "linux1804-64-clang-trunk-qr/opt" in task_label
:
1051 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
.label
)]
1054 # Run Updatebot's cron job 4 times daily.
1055 @_target_task("updatebot_cron")
1056 def target_tasks_updatebot_cron(full_task_graph
, parameters
, graph_config
):
1057 """Select tasks required to run Updatebot's cron job"""
1058 return ["updatebot-cron"]
1061 @_target_task("customv8_update")
1062 def target_tasks_customv8_update(full_task_graph
, parameters
, graph_config
):
1063 """Select tasks required for building latest d8/v8 version."""
1064 return ["toolchain-linux64-custom-v8"]
1067 @_target_task("chromium_update")
1068 def target_tasks_chromium_update(full_task_graph
, parameters
, graph_config
):
1069 """Select tasks required for building latest chromium versions."""
1071 "fetch-linux64-chromium",
1072 "fetch-win32-chromium",
1073 "fetch-win64-chromium",
1074 "fetch-mac-chromium",
1075 "toolchain-linux64-custom-car",
1076 "toolchain-win64-custom-car",
1077 "toolchain-macosx64-custom-car",
1081 @_target_task("file_update")
1082 def target_tasks_file_update(full_task_graph
, parameters
, graph_config
):
1083 """Select the set of tasks required to perform nightly in-tree file updates"""
1086 # For now any task in the repo-update kind is ok
1087 return task
.kind
in ["repo-update"]
1089 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1092 @_target_task("l10n_bump")
1093 def target_tasks_l10n_bump(full_task_graph
, parameters
, graph_config
):
1094 """Select the set of tasks required to perform l10n bumping."""
1097 # For now any task in the repo-update kind is ok
1098 return task
.kind
in ["l10n-bump"]
1100 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1103 @_target_task("merge_automation")
1104 def target_tasks_merge_automation(full_task_graph
, parameters
, graph_config
):
1105 """Select the set of tasks required to perform repository merges."""
1108 # For now any task in the repo-update kind is ok
1109 return task
.kind
in ["merge-automation"]
1111 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1114 @_target_task("scriptworker_canary")
1115 def target_tasks_scriptworker_canary(full_task_graph
, parameters
, graph_config
):
1116 """Select the set of tasks required to run scriptworker canaries."""
1119 # For now any task in the repo-update kind is ok
1120 return task
.kind
in ["scriptworker-canary"]
1122 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1125 @_target_task("cron_bouncer_check")
1126 def target_tasks_bouncer_check(full_task_graph
, parameters
, graph_config
):
1127 """Select the set of tasks required to perform bouncer version verification."""
1130 if not filter_for_project(task
, parameters
):
1132 # For now any task in the repo-update kind is ok
1133 return task
.kind
in ["cron-bouncer-check"]
1135 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1138 @_target_task("staging_release_builds")
1139 def target_tasks_staging_release(full_task_graph
, parameters
, graph_config
):
1141 Select all builds that are part of releases.
1145 if not task
.attributes
.get("shipping_product"):
1147 if parameters
["release_type"].startswith(
1149 ) and "android" in task
.attributes
.get("build_platform", ""):
1151 if parameters
["release_type"] != "beta" and "devedition" in task
.attributes
.get(
1152 "build_platform", ""
1155 if task
.attributes
.get("shipping_phase") == "build":
1159 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1162 @_target_task("release_simulation")
1163 def target_tasks_release_simulation(full_task_graph
, parameters
, graph_config
):
1165 Select builds that would run on push on a release branch.
1167 project_by_release
= {
1168 "nightly": "mozilla-central",
1169 "beta": "mozilla-beta",
1170 "release": "mozilla-release",
1171 "esr102": "mozilla-esr102",
1172 "esr115": "mozilla-esr115",
1174 target_project
= project_by_release
.get(parameters
["release_type"])
1175 if target_project
is None:
1176 raise Exception("Unknown or unspecified release type in simulation run.")
1178 def filter_for_target_project(task
):
1179 """Filter tasks by project. Optionally enable nightlies."""
1180 run_on_projects
= set(task
.attributes
.get("run_on_projects", []))
1181 return match_run_on_projects(target_project
, run_on_projects
)
1183 def filter_out_android_on_esr(task
):
1184 if parameters
["release_type"].startswith(
1186 ) and "android" in task
.attributes
.get("build_platform", ""):
1192 for l
, t
in full_task_graph
.tasks
.items()
1193 if filter_release_tasks(t
, parameters
)
1194 and filter_out_cron(t
, parameters
)
1195 and filter_for_target_project(t
)
1196 and filter_out_android_on_esr(t
)
1200 @_target_task("codereview")
1201 def target_tasks_codereview(full_task_graph
, parameters
, graph_config
):
1202 """Select all code review tasks needed to produce a report"""
1206 if task
.kind
in ["code-review"]:
1210 if task
.attributes
.get("code-review") is True:
1215 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1218 @_target_task("nothing")
1219 def target_tasks_nothing(full_task_graph
, parameters
, graph_config
):
1220 """Select nothing, for DONTBUILD pushes"""
1224 @_target_task("daily_beta_perf")
1225 def target_tasks_daily_beta_perf(full_task_graph
, parameters
, graph_config
):
1227 Select performance tests on the beta branch to be run daily
1230 f
"{graph_config['trust-domain']}.v2.{parameters['project']}.revision."
1231 f
"{parameters['head_rev']}.taskgraph.decision-daily-beta-perf"
1233 if os
.environ
.get("MOZ_AUTOMATION") and retry(
1237 "reason": "to avoid triggering multiple daily beta perftests off of the same revision",
1243 platform
= task
.attributes
.get("test_platform")
1244 attributes
= task
.attributes
1245 try_name
= attributes
.get("raptor_try_name")
1247 if attributes
.get("unittest_suite") != "raptor":
1250 if platform
and accept_raptor_android_build(platform
):
1251 # Select browsertime & geckoview specific tests
1252 if "browsertime" and "geckoview" in try_name
:
1253 if "g5" in platform
:
1255 if "power" in try_name
:
1257 if "cpu" in try_name
:
1259 if "profiling" in try_name
:
1261 if "-live" in try_name
:
1263 if "speedometer" in try_name
:
1265 if "webgl" in try_name
:
1267 if "tp6m" in try_name
:
1272 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1275 @_target_task("weekly_release_perf")
1276 def target_tasks_weekly_release_perf(full_task_graph
, parameters
, graph_config
):
1278 Select performance tests on the release branch to be run weekly
1282 platform
= task
.attributes
.get("test_platform")
1283 attributes
= task
.attributes
1284 try_name
= attributes
.get("raptor_try_name")
1286 if attributes
.get("unittest_suite") != "raptor":
1289 if platform
and accept_raptor_android_build(platform
):
1290 # Select browsertime & geckoview specific tests
1291 if "browsertime" and "geckoview" in try_name
:
1292 if "g5" in platform
:
1294 if "power" in try_name
:
1296 if "cpu" in try_name
:
1298 if "profiling" in try_name
:
1300 if "-live" in try_name
:
1302 if "speedometer" in try_name
:
1304 if "webgl" in try_name
:
1306 if "tp6m" in try_name
:
1308 if "youtube-playback" in try_name
:
1313 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1316 @_target_task("raptor_tp6m")
1317 def target_tasks_raptor_tp6m(full_task_graph
, parameters
, graph_config
):
1319 Select tasks required for running raptor cold page-load tests on fenix and refbrow
1323 platform
= task
.attributes
.get("build_platform")
1324 attributes
= task
.attributes
1326 if platform
and "android" not in platform
:
1328 if attributes
.get("unittest_suite") != "raptor":
1330 try_name
= attributes
.get("raptor_try_name")
1331 if "-cold" in try_name
and "shippable" in platform
:
1332 # Get browsertime amazon smoke tests
1334 "browsertime" in try_name
1335 and "amazon" in try_name
1336 and "search" not in try_name
1337 and "fenix" in try_name
1341 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1344 @_target_task("backfill_all_browsertime")
1345 def target_tasks_backfill_all_browsertime(full_task_graph
, parameters
, graph_config
):
1347 Search for revisions that contains patches that were reviewed by perftest reviewers
1348 and landed the day before the cron is running. Trigger backfill-all-browsertime action
1349 task on each of them.
1351 from gecko_taskgraph
.actions
.util
import get_decision_task_id
, get_pushes
1353 def date_is_yesterday(date
):
1354 yesterday
= datetime
.today() - timedelta(days
=1)
1355 date
= datetime
.fromtimestamp(date
)
1356 return date
.date() == yesterday
.date()
1358 def reviewed_by_perftest(push
):
1360 commit_message
= get_hg_commit_message(
1361 os
.path
.join(GECKO
, graph_config
["product-dir"]), rev
=push
1363 except Exception as e
:
1367 for line
in commit_message
.split("\n\n"):
1368 if line
.lower().startswith("bug ") and "r=" in line
:
1369 if "perftest-reviewers" in line
.split("r=")[-1]:
1374 pushes
= get_pushes(
1375 project
=parameters
["head_repository"],
1376 end_id
=int(parameters
["pushlog_id"]),
1380 for push_id
in sorted([int(p
) for p
in pushes
.keys()], reverse
=True):
1381 push_rev
= pushes
[str(push_id
)]["changesets"][-1]
1382 push_info
= find_hg_revision_push_info(
1383 "https://hg.mozilla.org/integration/" + parameters
["project"], push_rev
1385 pushdate
= int(push_info
["pushdate"])
1386 if date_is_yesterday(pushdate
) and reviewed_by_perftest(push_rev
):
1387 from gecko_taskgraph
.actions
.util
import trigger_action
1390 f
"Revision {push_rev} was created yesterday and was reviewed by "
1391 f
"#perftest-reviewers."
1394 push_decision_task_id
= get_decision_task_id(
1395 parameters
["project"], push_id
1398 print(f
"Could not find decision task for push {push_id}")
1402 action_name
="backfill-all-browsertime",
1403 # This lets the action know on which push we want to add a new task
1404 decision_task_id
=push_decision_task_id
,
1406 except Exception as e
:
1407 print(f
"Failed to trigger action for {push_rev}: {e}")
1412 @_target_task("condprof")
1413 def target_tasks_condprof(full_task_graph
, parameters
, graph_config
):
1415 Select tasks required for building conditioned profiles.
1417 for name
, task
in full_task_graph
.tasks
.items():
1418 if task
.kind
== "condprof":
1419 if "a51" not in name
: # bug 1765348
1423 @_target_task("system_symbols")
1424 def target_tasks_system_symbols(full_task_graph
, parameters
, graph_config
):
1426 Select tasks for scraping and uploading system symbols.
1428 for name
, task
in full_task_graph
.tasks
.items():
1431 "system-symbols-upload",
1432 "system-symbols-reprocess",
1437 @_target_task("perftest")
1438 def target_tasks_perftest(full_task_graph
, parameters
, graph_config
):
1440 Select perftest tasks we want to run daily
1442 for name
, task
in full_task_graph
.tasks
.items():
1443 if task
.kind
!= "perftest":
1445 if task
.attributes
.get("cron", False):
1449 @_target_task("perftest-on-autoland")
1450 def target_tasks_perftest_autoland(full_task_graph
, parameters
, graph_config
):
1452 Select perftest tasks we want to run daily
1454 for name
, task
in full_task_graph
.tasks
.items():
1455 if task
.kind
!= "perftest":
1457 if task
.attributes
.get("cron", False) and any(
1458 test_name
in name
for test_name
in ["view"]
1463 @_target_task("l10n-cross-channel")
1464 def target_tasks_l10n_cross_channel(full_task_graph
, parameters
, graph_config
):
1465 """Select the set of tasks required to run l10n cross-channel."""
1468 return task
.kind
in ["l10n-cross-channel"]
1470 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1473 @_target_task("are-we-esmified-yet")
1474 def target_tasks_are_we_esmified_yet(full_task_graph
, parameters
, graph_config
):
1476 select the task to track the progress of the esmification project
1479 l
for l
, t
in full_task_graph
.tasks
.items() if t
.kind
== "are-we-esmified-yet"
1483 @_target_task("eslint-build")
1484 def target_tasks_eslint_build(full_task_graph
, parameters
, graph_config
):
1485 """Select the task to run additional ESLint rules which require a build."""
1487 for name
, task
in full_task_graph
.tasks
.items():
1488 if task
.kind
!= "source-test":
1490 if "eslint-build" in name
:
1494 @_target_task("holly_tasks")
1495 def target_tasks_holly(full_task_graph
, parameters
, graph_config
):
1496 """Bug 1814661: only run updatebot tasks on holly"""
1499 return task
.kind
== "updatebot"
1501 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1504 @_target_task("snap_upstream_build")
1505 def target_tasks_snap_upstream_build(full_task_graph
, parameters
, graph_config
):
1507 Select tasks for building snap as upstream. Omit -try because it does not
1508 really make sense on m-c
1510 for name
, task
in full_task_graph
.tasks
.items():
1511 if "snap-upstream-build" in name
and not "-try" in name
: