1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 from datetime
import datetime
, timedelta
11 from redo
import retry
12 from taskgraph
.parameters
import Parameters
13 from taskgraph
.target_tasks
import _target_task
, get_method
14 from taskgraph
.util
.taskcluster
import find_task_id
16 from gecko_taskgraph
import GECKO
, try_option_syntax
17 from gecko_taskgraph
.util
.attributes
import (
18 match_run_on_hg_branches
,
19 match_run_on_projects
,
21 from gecko_taskgraph
.util
.hg
import find_hg_revision_push_info
, get_hg_commit_message
22 from gecko_taskgraph
.util
.platforms
import platform_family
24 # Some tasks show up in the target task set, but are possibly special cases,
25 # uncommon tasks, or tasks running against limited hardware set that they
26 # should only be selectable with --full.
27 UNCOMMON_TRY_TASK_LABELS
= [
28 # Platforms and/or Build types
29 r
"build-.*-gcp", # Bug 1631990
30 r
"mingwclang", # Bug 1631990
31 r
"valgrind", # Bug 1631990
33 r
"android-geckoview-docs",
36 r
"windows10-64-ref-hw",
37 r
"windows10-aarch64-qr",
39 r
"linux-", # hide all linux32 tasks by default - bug 1599197
40 r
"linux1804-32", # hide linux32 tests - bug 1599197
42 r
"web-platform-tests.*backlog", # hide wpt jobs that are not implemented yet - bug 1572820
44 r
"-profiling-", # talos/raptor profiling jobs are run too often
45 r
"-32-.*-webgpu", # webgpu gets little benefit from these tests.
48 # Hide shippable versions of tests we have opt versions of because the non-shippable
49 # versions are faster to run. This is mostly perf tests.
50 r
"-shippable(?!.*(awsy|browsertime|marionette-headless|mochitest-devtools-chrome-fis|raptor|talos|web-platform-tests-wdspec-headless|mochitest-plain-headless))", # noqa - too long
54 def index_exists(index_path
, reason
=""):
55 print(f
"Looking for existing index {index_path} {reason}...")
57 task_id
= find_task_id(index_path
)
58 print(f
"Index {index_path} exists: taskId {task_id}")
61 print(f
"Index {index_path} doesn't exist.")
65 def filter_out_shipping_phase(task
, parameters
):
67 # nightly still here because of geckodriver
68 not task
.attributes
.get("nightly")
69 and task
.attributes
.get("shipping_phase") in (None, "build")
73 def filter_out_devedition(task
, parameters
):
74 return not task
.attributes
.get("shipping_product") == "devedition"
77 def filter_out_cron(task
, parameters
):
79 Filter out tasks that run via cron.
81 return not task
.attributes
.get("cron")
84 def filter_for_project(task
, parameters
):
85 """Filter tasks by project. Optionally enable nightlies."""
86 run_on_projects
= set(task
.attributes
.get("run_on_projects", []))
87 return match_run_on_projects(parameters
["project"], run_on_projects
)
90 def filter_for_hg_branch(task
, parameters
):
91 """Filter tasks by hg branch.
92 If `run_on_hg_branch` is not defined, then task runs on all branches"""
93 run_on_hg_branches
= set(task
.attributes
.get("run_on_hg_branches", ["all"]))
94 return match_run_on_hg_branches(parameters
["hg_branch"], run_on_hg_branches
)
97 def filter_on_platforms(task
, platforms
):
98 """Filter tasks on the given platform"""
99 platform
= task
.attributes
.get("build_platform")
100 return platform
in platforms
103 def filter_by_uncommon_try_tasks(task
, optional_filters
=None):
104 """Filters tasks that should not be commonly run on try.
107 task (str): String representing the task name.
108 optional_filters (list, optional):
109 Additional filters to apply to task filtering.
112 (Boolean): True if task does not match any known filters.
115 filters
= UNCOMMON_TRY_TASK_LABELS
117 filters
= itertools
.chain(filters
, optional_filters
)
119 return not any(re
.search(pattern
, task
) for pattern
in filters
)
122 def filter_by_regex(task_label
, regexes
, mode
="include"):
123 """Filters tasks according to a list of pre-compiled reguar expressions.
125 If mode is "include", a task label must match any regex to pass.
126 If it is "exclude", a task label must _not_ match any regex to pass.
131 assert mode
in ["include", "exclude"]
133 any_match
= any(r
.search(task_label
) for r
in regexes
)
135 return mode
== "include"
136 return mode
!= "include"
139 def filter_release_tasks(task
, parameters
):
140 platform
= task
.attributes
.get("build_platform")
149 if task
.attributes
["kind"] == "l10n":
150 # This is on-change l10n
153 task
.attributes
["build_type"] == "opt"
154 and task
.attributes
.get("unittest_suite") != "talos"
155 and task
.attributes
.get("unittest_suite") != "raptor"
159 if task
.attributes
.get("shipping_phase") not in (None, "build"):
162 """ No debug on release, keep on ESR with 4 week cycles, release
163 will not be too different from central, but ESR will live for a long time.
165 From June 2019 -> June 2020, we found 1 unique regression on ESR debug
166 and 5 unique regressions on beta/release. Keeping spidermonkey and linux
167 debug finds all but 1 unique regressions (windows found on try) for beta/release.
169 ...but debug-only failures started showing up on ESR (esr-91, esr-102) so
170 desktop debug tests were added back for beta.
172 build_type
= task
.attributes
.get("build_type", "")
173 build_platform
= task
.attributes
.get("build_platform", "")
174 test_platform
= task
.attributes
.get("test_platform", "")
176 if parameters
["release_type"].startswith("esr") or (
177 parameters
["release_type"] == "beta" and "android" not in build_platform
181 # code below here is intended to reduce release debug tasks
182 if task
.kind
== "hazard" or "toolchain" in build_platform
:
183 # keep hazard and toolchain builds around
186 if build_type
== "debug":
187 if "linux" not in build_platform
:
188 # filter out windows/mac/android
190 if task
.kind
not in ["spidermonkey"] and "-qr" in test_platform
:
191 # filter out linux-qr tests, leave spidermonkey
193 if "64" not in build_platform
:
194 # filter out linux32 builds
197 # webrender-android-*-debug doesn't have attributes to find 'debug', using task.label.
198 if task
.kind
== "webrender" and "debug" in task
.label
:
203 def filter_out_missing_signoffs(task
, parameters
):
204 for signoff
in parameters
["required_signoffs"]:
205 if signoff
not in parameters
["signoff_urls"] and signoff
in task
.attributes
.get(
206 "required_signoffs", []
212 def filter_tests_without_manifests(task
, parameters
):
213 """Remove test tasks that have an empty 'test_manifests' attribute.
215 This situation can arise when the test loader (e.g bugbug) decided there
216 weren't any important manifests to run for the given push. We filter tasks
217 out here rather than in the transforms so that the full task graph is still
218 aware that the task exists (which is needed by the backfill action).
222 and "test_manifests" in task
.attributes
223 and not task
.attributes
["test_manifests"]
229 def standard_filter(task
, parameters
):
231 filter_func(task
, parameters
)
235 filter_for_hg_branch
,
236 filter_tests_without_manifests
,
241 def accept_raptor_android_build(platform
):
242 """Helper function for selecting the correct android raptor builds."""
243 if "android" not in platform
:
245 if "shippable" not in platform
:
247 if "p2" in platform
and "aarch64" in platform
:
249 if "p5" in platform
and "aarch64" in platform
:
253 if "a51" in platform
:
257 def filter_unsupported_artifact_builds(task
, parameters
):
258 try_config
= parameters
.get("try_task_config", {})
259 if not try_config
.get("use-artifact-builds", False):
262 supports_artifact_builds
= task
.attributes
.get("supports-artifact-builds", True)
263 return supports_artifact_builds
266 def filter_out_shippable(task
):
267 return not task
.attributes
.get("shippable", False)
270 def _try_task_config(full_task_graph
, parameters
, graph_config
):
271 requested_tasks
= parameters
["try_task_config"]["tasks"]
272 return list(set(requested_tasks
) & full_task_graph
.graph
.nodes
)
275 def _try_option_syntax(full_task_graph
, parameters
, graph_config
):
276 """Generate a list of target tasks based on try syntax in
277 parameters['message'] and, for context, the full task graph."""
278 options
= try_option_syntax
.TryOptionSyntax(
279 parameters
, full_task_graph
, graph_config
281 target_tasks_labels
= [
283 for t
in full_task_graph
.tasks
.values()
284 if options
.task_matches(t
)
285 and filter_by_uncommon_try_tasks(t
.label
)
286 and filter_unsupported_artifact_builds(t
, parameters
)
290 k
: getattr(options
, k
)
297 for l
in target_tasks_labels
:
298 task
= full_task_graph
[l
]
299 if "unittest_suite" in task
.attributes
:
300 task
.attributes
["task_duplicates"] = options
.trigger_tests
302 for l
in target_tasks_labels
:
303 task
= full_task_graph
[l
]
304 # If the developer wants test jobs to be rebuilt N times we add that value here
305 if options
.trigger_tests
> 1 and "unittest_suite" in task
.attributes
:
306 task
.attributes
["task_duplicates"] = options
.trigger_tests
308 # If the developer wants test talos jobs to be rebuilt N times we add that value here
310 options
.talos_trigger_tests
> 1
311 and task
.attributes
.get("unittest_suite") == "talos"
313 task
.attributes
["task_duplicates"] = options
.talos_trigger_tests
315 # If the developer wants test raptor jobs to be rebuilt N times we add that value here
317 options
.raptor_trigger_tests
318 and options
.raptor_trigger_tests
> 1
319 and task
.attributes
.get("unittest_suite") == "raptor"
321 task
.attributes
["task_duplicates"] = options
.raptor_trigger_tests
323 task
.attributes
.update(attributes
)
325 # Add notifications here as well
326 if options
.notifications
:
327 for task
in full_task_graph
:
328 owner
= parameters
.get("owner")
329 routes
= task
.task
.setdefault("routes", [])
330 if options
.notifications
== "all":
331 routes
.append(f
"notify.email.{owner}.on-any")
332 elif options
.notifications
== "failure":
333 routes
.append(f
"notify.email.{owner}.on-failed")
334 routes
.append(f
"notify.email.{owner}.on-exception")
336 return target_tasks_labels
339 @_target_task("try_tasks")
340 def target_tasks_try(full_task_graph
, parameters
, graph_config
):
341 try_mode
= parameters
["try_mode"]
342 if try_mode
== "try_task_config":
343 return _try_task_config(full_task_graph
, parameters
, graph_config
)
344 if try_mode
== "try_option_syntax":
345 return _try_option_syntax(full_task_graph
, parameters
, graph_config
)
346 # With no try mode, we schedule nothing, allowing the user to add tasks
347 # later via treeherder.
351 @_target_task("try_select_tasks")
352 def target_tasks_try_select(full_task_graph
, parameters
, graph_config
):
353 tasks
= target_tasks_try_select_uncommon(full_task_graph
, parameters
, graph_config
)
354 return [l
for l
in tasks
if filter_by_uncommon_try_tasks(l
)]
357 @_target_task("try_select_tasks_uncommon")
358 def target_tasks_try_select_uncommon(full_task_graph
, parameters
, graph_config
):
359 from gecko_taskgraph
.decision
import PER_PROJECT_PARAMETERS
361 projects
= ("autoland", "mozilla-central")
362 if parameters
["project"] not in projects
:
363 projects
= (parameters
["project"],)
366 for project
in projects
:
367 params
= dict(parameters
)
368 params
["project"] = project
369 parameters
= Parameters(**params
)
372 target_tasks_method
= PER_PROJECT_PARAMETERS
[project
]["target_tasks_method"]
374 target_tasks_method
= "default"
377 get_method(target_tasks_method
)(full_task_graph
, parameters
, graph_config
)
383 @_target_task("try_auto")
384 def target_tasks_try_auto(full_task_graph
, parameters
, graph_config
):
385 """Target the tasks which have indicated they should be run on autoland
386 (rather than try) via the `run_on_projects` attributes.
388 Should do the same thing as the `default` target tasks method.
390 params
= dict(parameters
)
391 params
["project"] = "autoland"
392 parameters
= Parameters(**params
)
394 regex_filters
= parameters
["try_task_config"].get("tasks-regex")
395 include_regexes
= exclude_regexes
= []
397 include_regexes
= [re
.compile(r
) for r
in regex_filters
.get("include", [])]
398 exclude_regexes
= [re
.compile(r
) for r
in regex_filters
.get("exclude", [])]
402 for l
, t
in full_task_graph
.tasks
.items()
403 if standard_filter(t
, parameters
)
404 and filter_out_shipping_phase(t
, parameters
)
405 and filter_out_devedition(t
, parameters
)
406 and filter_by_uncommon_try_tasks(t
.label
)
407 and filter_by_regex(t
.label
, include_regexes
, mode
="include")
408 and filter_by_regex(t
.label
, exclude_regexes
, mode
="exclude")
409 and filter_unsupported_artifact_builds(t
, parameters
)
410 and filter_out_shippable(t
)
414 @_target_task("default")
415 def target_tasks_default(full_task_graph
, parameters
, graph_config
):
416 """Target the tasks which have indicated they should be run on this project
417 via the `run_on_projects` attributes."""
420 for l
, t
in full_task_graph
.tasks
.items()
421 if standard_filter(t
, parameters
)
422 and filter_out_shipping_phase(t
, parameters
)
423 and filter_out_devedition(t
, parameters
)
427 @_target_task("autoland_tasks")
428 def target_tasks_autoland(full_task_graph
, parameters
, graph_config
):
429 """In addition to doing the filtering by project that the 'default'
430 filter does, also remove any tests running against shippable builds
431 for non-backstop pushes."""
432 filtered_for_project
= target_tasks_default(
433 full_task_graph
, parameters
, graph_config
437 if task
.kind
!= "test":
440 if parameters
["backstop"]:
443 build_type
= task
.attributes
.get("build_type")
445 if not build_type
or build_type
!= "opt" or filter_out_shippable(task
):
450 return [l
for l
in filtered_for_project
if filter(full_task_graph
[l
])]
453 @_target_task("mozilla_central_tasks")
454 def target_tasks_mozilla_central(full_task_graph
, parameters
, graph_config
):
455 """In addition to doing the filtering by project that the 'default'
456 filter does, also remove any tests running against regular (aka not shippable,
457 asan, etc.) opt builds."""
458 filtered_for_project
= target_tasks_default(
459 full_task_graph
, parameters
, graph_config
463 if task
.kind
!= "test":
466 build_platform
= task
.attributes
.get("build_platform")
467 build_type
= task
.attributes
.get("build_type")
468 shippable
= task
.attributes
.get("shippable", False)
470 if not build_platform
or not build_type
:
473 family
= platform_family(build_platform
)
474 # We need to know whether this test is against a "regular" opt build
475 # (which is to say, not shippable, asan, tsan, or any other opt build
476 # with other properties). There's no positive test for this, so we have to
477 # do it somewhat hackily. Android doesn't have variants other than shippable
478 # so it is pretty straightforward to check for. Other platforms have many
479 # variants, but none of the regular opt builds we're looking for have a "-"
480 # in their platform name, so this works (for now).
482 family
== "android" and not shippable
483 ) or "-" not in build_platform
485 if build_type
!= "opt" or not is_regular_opt
:
490 return [l
for l
in filtered_for_project
if filter(full_task_graph
[l
])]
493 @_target_task("graphics_tasks")
494 def target_tasks_graphics(full_task_graph
, parameters
, graph_config
):
495 """In addition to doing the filtering by project that the 'default'
496 filter does, also remove artifact builds because we have csets on
497 the graphics branch that aren't on the candidate branches of artifact
499 filtered_for_project
= target_tasks_default(
500 full_task_graph
, parameters
, graph_config
504 if task
.attributes
["kind"] == "artifact-build":
508 return [l
for l
in filtered_for_project
if filter(full_task_graph
[l
])]
511 @_target_task("mozilla_beta_tasks")
512 def target_tasks_mozilla_beta(full_task_graph
, parameters
, graph_config
):
513 """Select the set of tasks required for a promotable beta or release build
514 of desktop, plus android CI. The candidates build process involves a pipeline
515 of builds and signing, but does not include beetmover or balrog jobs."""
519 for l
, t
in full_task_graph
.tasks
.items()
520 if filter_release_tasks(t
, parameters
) and standard_filter(t
, parameters
)
524 @_target_task("mozilla_release_tasks")
525 def target_tasks_mozilla_release(full_task_graph
, parameters
, graph_config
):
526 """Select the set of tasks required for a promotable beta or release build
527 of desktop, plus android CI. The candidates build process involves a pipeline
528 of builds and signing, but does not include beetmover or balrog jobs."""
532 for l
, t
in full_task_graph
.tasks
.items()
533 if filter_release_tasks(t
, parameters
) and standard_filter(t
, parameters
)
537 @_target_task("mozilla_esr115_tasks")
538 def target_tasks_mozilla_esr115(full_task_graph
, parameters
, graph_config
):
539 """Select the set of tasks required for a promotable beta or release build
540 of desktop, without android CI. The candidates build process involves a pipeline
541 of builds and signing, but does not include beetmover or balrog jobs."""
544 if not filter_release_tasks(task
, parameters
):
547 if not standard_filter(task
, parameters
):
550 platform
= task
.attributes
.get("build_platform")
552 # Android is not built on esr115.
553 if platform
and "android" in platform
:
558 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
561 @_target_task("promote_desktop")
562 def target_tasks_promote_desktop(full_task_graph
, parameters
, graph_config
):
563 """Select the superset of tasks required to promote a beta or release build
564 of a desktop product. This should include all non-android
565 mozilla_{beta,release} tasks, plus l10n, beetmover, balrog, etc."""
568 # Bug 1758507 - geckoview ships in the promote phase
569 if not parameters
["release_type"].startswith("esr") and is_geckoview(
574 if task
.attributes
.get("shipping_product") != parameters
["release_product"]:
577 # 'secondary' balrog/update verify/final verify tasks only run for RCs
578 if parameters
.get("release_type") != "release-rc":
579 if "secondary" in task
.kind
:
582 if not filter_out_missing_signoffs(task
, parameters
):
585 if task
.attributes
.get("shipping_phase") == "promote":
588 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
591 def is_geckoview(task
, parameters
):
593 task
.attributes
.get("shipping_product") == "fennec"
594 and task
.kind
in ("beetmover-geckoview", "upload-symbols")
595 and parameters
["release_product"] == "firefox"
599 @_target_task("push_desktop")
600 def target_tasks_push_desktop(full_task_graph
, parameters
, graph_config
):
601 """Select the set of tasks required to push a build of desktop to cdns.
602 Previous build deps will be optimized out via action task."""
603 filtered_for_candidates
= target_tasks_promote_desktop(
610 if not filter_out_missing_signoffs(task
, parameters
):
612 # Include promotion tasks; these will be optimized out
613 if task
.label
in filtered_for_candidates
:
617 task
.attributes
.get("shipping_product") == parameters
["release_product"]
618 and task
.attributes
.get("shipping_phase") == "push"
622 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
625 @_target_task("ship_desktop")
626 def target_tasks_ship_desktop(full_task_graph
, parameters
, graph_config
):
627 """Select the set of tasks required to ship desktop.
628 Previous build deps will be optimized out via action task."""
629 is_rc
= parameters
.get("release_type") == "release-rc"
631 # ship_firefox_rc runs after `promote` rather than `push`; include
633 filtered_for_candidates
= target_tasks_promote_desktop(
639 # ship_firefox runs after `push`; include all push tasks.
640 filtered_for_candidates
= target_tasks_push_desktop(
647 if not filter_out_missing_signoffs(task
, parameters
):
649 # Include promotion tasks; these will be optimized out
650 if task
.label
in filtered_for_candidates
:
654 task
.attributes
.get("shipping_product") != parameters
["release_product"]
655 or task
.attributes
.get("shipping_phase") != "ship"
659 if "secondary" in task
.kind
:
663 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
666 @_target_task("pine_tasks")
667 def target_tasks_pine(full_task_graph
, parameters
, graph_config
):
668 """Bug 1339179 - no mobile automation needed on pine"""
671 platform
= task
.attributes
.get("build_platform")
672 # disable mobile jobs
673 if str(platform
).startswith("android"):
676 if platform
== "linux64-asan":
678 # disable non-pine and tasks with a shipping phase
679 if standard_filter(task
, parameters
) or filter_out_shipping_phase(
684 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
687 @_target_task("kaios_tasks")
688 def target_tasks_kaios(full_task_graph
, parameters
, graph_config
):
689 """The set of tasks to run for kaios integration"""
692 # We disable everything in central, and adjust downstream.
695 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
698 @_target_task("ship_geckoview")
699 def target_tasks_ship_geckoview(full_task_graph
, parameters
, graph_config
):
700 """Select the set of tasks required to ship geckoview nightly. The
701 nightly build process involves a pipeline of builds and an upload to
702 maven.mozilla.org."""
704 f
"{graph_config['trust-domain']}.v2.{parameters['project']}.revision."
705 f
"{parameters['head_rev']}.taskgraph.decision-ship-geckoview"
707 if os
.environ
.get("MOZ_AUTOMATION") and retry(
711 "reason": "to avoid triggering multiple nightlies off the same revision",
717 # XXX Starting 69, we don't ship Fennec Nightly anymore. We just want geckoview to be
719 return task
.attributes
.get("shipping_product") == "fennec" and task
.kind
in (
720 "beetmover-geckoview",
724 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
727 @_target_task("custom-car_perf_testing")
728 def target_tasks_custom_car_perf_testing(full_task_graph
, parameters
, graph_config
):
729 """Select tasks required for running daily performance tests for custom chromium-as-release."""
732 platform
= task
.attributes
.get("test_platform")
733 attributes
= task
.attributes
734 if attributes
.get("unittest_suite") != "raptor":
737 try_name
= attributes
.get("raptor_try_name")
739 # Completely ignore all non-shippable platforms
740 if "shippable" not in platform
:
743 # ignore all windows 7 perf jobs scheduled automatically
744 if "windows10-32" in platform
:
747 # Desktop selection only for CaR
748 if "android" not in platform
:
749 if "browsertime" in try_name
and "custom-car" in try_name
:
753 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
756 @_target_task("general_perf_testing")
757 def target_tasks_general_perf_testing(full_task_graph
, parameters
, graph_config
):
759 Select tasks required for running performance tests 3 times a week.
763 platform
= task
.attributes
.get("test_platform")
764 attributes
= task
.attributes
765 if attributes
.get("unittest_suite") != "raptor":
768 try_name
= attributes
.get("raptor_try_name")
770 # Completely ignore all non-shippable platforms
771 if "shippable" not in platform
:
774 # ignore all windows 7 perf jobs scheduled automatically
775 if "windows7" in platform
or "windows10-32" in platform
:
779 if "android" not in platform
:
780 # Select some browsertime tasks as desktop smoke-tests
781 if "browsertime" in try_name
:
782 if "chrome" in try_name
:
784 if "chromium" in try_name
:
786 # chromium-as-release has it's own cron
787 if "custom-car" in try_name
:
789 if "-live" in try_name
:
791 if "-fis" in try_name
:
793 if "linux" in platform
:
794 if "speedometer" in try_name
:
796 if "safari" and "benchmark" in try_name
:
797 # Speedometer 3 is broken on Safari, see bug 1802922
798 if "speedometer3" in try_name
:
802 # Don't run tp6 raptor tests
803 if "tp6" in try_name
:
806 elif accept_raptor_android_build(platform
):
807 if "chrome-m" in try_name
and (
808 ("ebay" in try_name
and "live" not in try_name
)
811 and ("facebook" in try_name
or "dailymail" in try_name
)
815 # Ignore all fennec tests here, we run those weekly
816 if "fennec" in try_name
:
818 # Only run webrender tests
819 if "chrome-m" not in try_name
and "-qr" not in platform
:
821 # Select live site tests
822 if "-live" in try_name
:
824 # Select fenix resource usage tests
825 if "fenix" in try_name
:
826 # Bug 1816421 disable fission perf tests
827 if "-fis" in try_name
:
829 if "-power" in try_name
:
831 # Select geckoview resource usage tests
832 if "geckoview" in try_name
:
833 # Bug 1816421 disable fission perf tests
834 if "-fis" in try_name
:
836 # Run cpu+memory, and power tests
837 cpu_n_memory_task
= "-cpu" in try_name
and "-memory" in try_name
838 power_task
= "-power" in try_name
839 # Ignore cpu+memory+power tests
840 if power_task
and cpu_n_memory_task
:
842 if cpu_n_memory_task
:
845 return "browsertime" in try_name
846 # Select browsertime-specific tests
847 if "browsertime" in try_name
:
848 if "speedometer" in try_name
:
852 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
855 def make_desktop_nightly_filter(platforms
):
856 """Returns a filter that gets all nightly tasks on the given platform."""
858 def filter(task
, parameters
):
861 filter_on_platforms(task
, platforms
),
862 filter_for_project(task
, parameters
),
863 task
.attributes
.get("shippable", False),
864 # Tests and nightly only builds don't have `shipping_product` set
865 task
.attributes
.get("shipping_product")
866 in {None, "firefox", "thunderbird"},
867 task
.kind
not in {"l10n"}, # no on-change l10n
874 @_target_task("sp-perftests")
875 def target_tasks_speedometer_tests(full_task_graph
, parameters
, graph_config
):
877 platform
= task
.attributes
.get("test_platform")
878 attributes
= task
.attributes
879 if attributes
.get("unittest_suite") != "raptor":
881 if "windows10-32" not in platform
:
882 try_name
= attributes
.get("raptor_try_name")
884 "browsertime" in try_name
885 and "speedometer" in try_name
886 and "chrome" in try_name
890 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
893 @_target_task("nightly_linux")
894 def target_tasks_nightly_linux(full_task_graph
, parameters
, graph_config
):
895 """Select the set of tasks required for a nightly build of linux. The
896 nightly build process involves a pipeline of builds, signing,
897 and, eventually, uploading the tasks to balrog."""
898 filter = make_desktop_nightly_filter({"linux64-shippable", "linux-shippable"})
899 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
902 @_target_task("nightly_macosx")
903 def target_tasks_nightly_macosx(full_task_graph
, parameters
, graph_config
):
904 """Select the set of tasks required for a nightly build of macosx. The
905 nightly build process involves a pipeline of builds, signing,
906 and, eventually, uploading the tasks to balrog."""
907 filter = make_desktop_nightly_filter({"macosx64-shippable"})
908 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
911 @_target_task("nightly_win32")
912 def target_tasks_nightly_win32(full_task_graph
, parameters
, graph_config
):
913 """Select the set of tasks required for a nightly build of win32 and win64.
914 The nightly build process involves a pipeline of builds, signing,
915 and, eventually, uploading the tasks to balrog."""
916 filter = make_desktop_nightly_filter({"win32-shippable"})
917 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
920 @_target_task("nightly_win64")
921 def target_tasks_nightly_win64(full_task_graph
, parameters
, graph_config
):
922 """Select the set of tasks required for a nightly build of win32 and win64.
923 The nightly build process involves a pipeline of builds, signing,
924 and, eventually, uploading the tasks to balrog."""
925 filter = make_desktop_nightly_filter({"win64-shippable"})
926 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
929 @_target_task("nightly_win64_aarch64")
930 def target_tasks_nightly_win64_aarch64(full_task_graph
, parameters
, graph_config
):
931 """Select the set of tasks required for a nightly build of win32 and win64.
932 The nightly build process involves a pipeline of builds, signing,
933 and, eventually, uploading the tasks to balrog."""
934 filter = make_desktop_nightly_filter({"win64-aarch64-shippable"})
935 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
938 @_target_task("nightly_asan")
939 def target_tasks_nightly_asan(full_task_graph
, parameters
, graph_config
):
940 """Select the set of tasks required for a nightly build of asan. The
941 nightly build process involves a pipeline of builds, signing,
942 and, eventually, uploading the tasks to balrog."""
943 filter = make_desktop_nightly_filter(
944 {"linux64-asan-reporter-shippable", "win64-asan-reporter-shippable"}
946 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
, parameters
)]
949 @_target_task("daily_releases")
950 def target_tasks_daily_releases(full_task_graph
, parameters
, graph_config
):
951 """Select the set of tasks required to identify if we should release.
952 If we determine that we should the task will communicate to ship-it to
953 schedule the release itself."""
956 return task
.kind
in ["maybe-release"]
958 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
961 @_target_task("nightly_desktop")
962 def target_tasks_nightly_desktop(full_task_graph
, parameters
, graph_config
):
963 """Select the set of tasks required for a nightly build of linux, mac,
966 f
"{graph_config['trust-domain']}.v2.{parameters['project']}.revision."
967 f
"{parameters['head_rev']}.taskgraph.decision-nightly-desktop"
969 if os
.environ
.get("MOZ_AUTOMATION") and retry(
973 "reason": "to avoid triggering multiple nightlies off the same revision",
978 # Tasks that aren't platform specific
979 release_filter
= make_desktop_nightly_filter({None}
)
981 l
for l
, t
in full_task_graph
.tasks
.items() if release_filter(t
, parameters
)
983 # Avoid duplicate tasks.
985 set(target_tasks_nightly_win32(full_task_graph
, parameters
, graph_config
))
986 |
set(target_tasks_nightly_win64(full_task_graph
, parameters
, graph_config
))
988 target_tasks_nightly_win64_aarch64(
989 full_task_graph
, parameters
, graph_config
992 |
set(target_tasks_nightly_macosx(full_task_graph
, parameters
, graph_config
))
993 |
set(target_tasks_nightly_linux(full_task_graph
, parameters
, graph_config
))
994 |
set(target_tasks_nightly_asan(full_task_graph
, parameters
, graph_config
))
999 # Run Searchfox analysis once daily.
1000 @_target_task("searchfox_index")
1001 def target_tasks_searchfox(full_task_graph
, parameters
, graph_config
):
1002 """Select tasks required for indexing Firefox for Searchfox web site each day"""
1004 "searchfox-linux64-searchfox/debug",
1005 "searchfox-macosx64-searchfox/debug",
1006 "searchfox-win64-searchfox/debug",
1007 "searchfox-android-armv7-searchfox/debug",
1008 "source-test-file-metadata-bugzilla-components",
1009 "source-test-file-metadata-test-info-all",
1010 "source-test-wpt-metadata-summary",
1014 # Run build linux64-plain-clang-trunk/opt on mozilla-central/beta with perf tests
1015 @_target_task("linux64_clang_trunk_perf")
1016 def target_tasks_build_linux64_clang_trunk_perf(
1017 full_task_graph
, parameters
, graph_config
1019 """Select tasks required to run perf test on linux64 build with clang trunk"""
1021 # Only keep tasks generated from platform `linux1804-64-clang-trunk-qr/opt`
1022 def filter(task_label
):
1023 if "linux1804-64-clang-trunk-qr/opt" in task_label
:
1027 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
.label
)]
1030 # Run Updatebot's cron job 4 times daily.
1031 @_target_task("updatebot_cron")
1032 def target_tasks_updatebot_cron(full_task_graph
, parameters
, graph_config
):
1033 """Select tasks required to run Updatebot's cron job"""
1034 return ["updatebot-cron"]
1037 @_target_task("customv8_update")
1038 def target_tasks_customv8_update(full_task_graph
, parameters
, graph_config
):
1039 """Select tasks required for building latest d8/v8 version."""
1040 return ["toolchain-linux64-custom-v8"]
1043 @_target_task("chromium_update")
1044 def target_tasks_chromium_update(full_task_graph
, parameters
, graph_config
):
1045 """Select tasks required for building latest chromium versions."""
1047 "fetch-linux64-chromium",
1048 "fetch-win32-chromium",
1049 "fetch-win64-chromium",
1050 "fetch-mac-chromium",
1051 "toolchain-linux64-custom-car",
1052 "toolchain-win64-custom-car",
1053 "toolchain-macosx64-custom-car",
1057 @_target_task("file_update")
1058 def target_tasks_file_update(full_task_graph
, parameters
, graph_config
):
1059 """Select the set of tasks required to perform nightly in-tree file updates"""
1062 # For now any task in the repo-update kind is ok
1063 return task
.kind
in ["repo-update"]
1065 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1068 @_target_task("l10n_bump")
1069 def target_tasks_l10n_bump(full_task_graph
, parameters
, graph_config
):
1070 """Select the set of tasks required to perform l10n bumping."""
1073 # For now any task in the repo-update kind is ok
1074 return task
.kind
in ["l10n-bump"]
1076 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1079 @_target_task("merge_automation")
1080 def target_tasks_merge_automation(full_task_graph
, parameters
, graph_config
):
1081 """Select the set of tasks required to perform repository merges."""
1084 # For now any task in the repo-update kind is ok
1085 return task
.kind
in ["merge-automation"]
1087 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1090 @_target_task("scriptworker_canary")
1091 def target_tasks_scriptworker_canary(full_task_graph
, parameters
, graph_config
):
1092 """Select the set of tasks required to run scriptworker canaries."""
1095 # For now any task in the repo-update kind is ok
1096 return task
.kind
in ["scriptworker-canary"]
1098 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1101 @_target_task("cron_bouncer_check")
1102 def target_tasks_bouncer_check(full_task_graph
, parameters
, graph_config
):
1103 """Select the set of tasks required to perform bouncer version verification."""
1106 if not filter_for_project(task
, parameters
):
1108 # For now any task in the repo-update kind is ok
1109 return task
.kind
in ["cron-bouncer-check"]
1111 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1114 @_target_task("staging_release_builds")
1115 def target_tasks_staging_release(full_task_graph
, parameters
, graph_config
):
1117 Select all builds that are part of releases.
1121 if not task
.attributes
.get("shipping_product"):
1123 if parameters
["release_type"].startswith(
1125 ) and "android" in task
.attributes
.get("build_platform", ""):
1127 if parameters
["release_type"] != "beta" and "devedition" in task
.attributes
.get(
1128 "build_platform", ""
1131 if task
.attributes
.get("shipping_phase") == "build":
1135 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1138 @_target_task("release_simulation")
1139 def target_tasks_release_simulation(full_task_graph
, parameters
, graph_config
):
1141 Select builds that would run on push on a release branch.
1143 project_by_release
= {
1144 "nightly": "mozilla-central",
1145 "beta": "mozilla-beta",
1146 "release": "mozilla-release",
1147 "esr115": "mozilla-esr115",
1149 target_project
= project_by_release
.get(parameters
["release_type"])
1150 if target_project
is None:
1151 raise Exception("Unknown or unspecified release type in simulation run.")
1153 def filter_for_target_project(task
):
1154 """Filter tasks by project. Optionally enable nightlies."""
1155 run_on_projects
= set(task
.attributes
.get("run_on_projects", []))
1156 return match_run_on_projects(target_project
, run_on_projects
)
1158 def filter_out_android_on_esr(task
):
1159 if parameters
["release_type"].startswith(
1161 ) and "android" in task
.attributes
.get("build_platform", ""):
1167 for l
, t
in full_task_graph
.tasks
.items()
1168 if filter_release_tasks(t
, parameters
)
1169 and filter_out_cron(t
, parameters
)
1170 and filter_for_target_project(t
)
1171 and filter_out_android_on_esr(t
)
1175 @_target_task("codereview")
1176 def target_tasks_codereview(full_task_graph
, parameters
, graph_config
):
1177 """Select all code review tasks needed to produce a report"""
1181 if task
.kind
in ["code-review"]:
1185 if task
.attributes
.get("code-review") is True:
1190 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1193 @_target_task("nothing")
1194 def target_tasks_nothing(full_task_graph
, parameters
, graph_config
):
1195 """Select nothing, for DONTBUILD pushes"""
1199 @_target_task("daily_beta_perf")
1200 def target_tasks_daily_beta_perf(full_task_graph
, parameters
, graph_config
):
1202 Select performance tests on the beta branch to be run daily
1205 f
"{graph_config['trust-domain']}.v2.{parameters['project']}.revision."
1206 f
"{parameters['head_rev']}.taskgraph.decision-daily-beta-perf"
1208 if os
.environ
.get("MOZ_AUTOMATION") and retry(
1212 "reason": "to avoid triggering multiple daily beta perftests off of the same revision",
1218 platform
= task
.attributes
.get("test_platform")
1219 attributes
= task
.attributes
1220 try_name
= attributes
.get("raptor_try_name")
1222 if attributes
.get("unittest_suite") != "raptor":
1225 if platform
and accept_raptor_android_build(platform
):
1226 # Select browsertime & geckoview specific tests
1227 if "browsertime" and "geckoview" in try_name
:
1228 if "g5" in platform
:
1230 if "power" in try_name
:
1232 if "cpu" in try_name
:
1234 if "profiling" in try_name
:
1236 if "-live" in try_name
:
1238 if "speedometer" in try_name
:
1240 if "webgl" in try_name
:
1242 if "tp6m" in try_name
:
1247 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1250 @_target_task("weekly_release_perf")
1251 def target_tasks_weekly_release_perf(full_task_graph
, parameters
, graph_config
):
1253 Select performance tests on the release branch to be run weekly
1257 platform
= task
.attributes
.get("test_platform")
1258 attributes
= task
.attributes
1259 try_name
= attributes
.get("raptor_try_name")
1261 if attributes
.get("unittest_suite") != "raptor":
1264 if platform
and accept_raptor_android_build(platform
):
1265 # Select browsertime & geckoview specific tests
1266 if "browsertime" and "geckoview" in try_name
:
1267 if "g5" in platform
:
1269 if "power" in try_name
:
1271 if "cpu" in try_name
:
1273 if "profiling" in try_name
:
1275 if "-live" in try_name
:
1277 if "speedometer" in try_name
:
1279 if "webgl" in try_name
:
1281 if "tp6m" in try_name
:
1283 if "youtube-playback" in try_name
:
1288 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1291 @_target_task("raptor_tp6m")
1292 def target_tasks_raptor_tp6m(full_task_graph
, parameters
, graph_config
):
1294 Select tasks required for running raptor cold page-load tests on fenix and refbrow
1298 platform
= task
.attributes
.get("build_platform")
1299 attributes
= task
.attributes
1301 if platform
and "android" not in platform
:
1303 if attributes
.get("unittest_suite") != "raptor":
1305 try_name
= attributes
.get("raptor_try_name")
1306 if "-cold" in try_name
and "shippable" in platform
:
1307 # Get browsertime amazon smoke tests
1309 "browsertime" in try_name
1310 and "amazon" in try_name
1311 and "search" not in try_name
1312 and "fenix" in try_name
1316 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1319 @_target_task("backfill_all_browsertime")
1320 def target_tasks_backfill_all_browsertime(full_task_graph
, parameters
, graph_config
):
1322 Search for revisions that contains patches that were reviewed by perftest reviewers
1323 and landed the day before the cron is running. Trigger backfill-all-browsertime action
1324 task on each of them.
1326 from gecko_taskgraph
.actions
.util
import get_decision_task_id
, get_pushes
1328 def date_is_yesterday(date
):
1329 yesterday
= datetime
.today() - timedelta(days
=1)
1330 date
= datetime
.fromtimestamp(date
)
1331 return date
.date() == yesterday
.date()
1333 def reviewed_by_perftest(push
):
1335 commit_message
= get_hg_commit_message(
1336 os
.path
.join(GECKO
, graph_config
["product-dir"]), rev
=push
1338 except Exception as e
:
1342 for line
in commit_message
.split("\n\n"):
1343 if line
.lower().startswith("bug ") and "r=" in line
:
1344 if "perftest-reviewers" in line
.split("r=")[-1]:
1349 pushes
= get_pushes(
1350 project
=parameters
["head_repository"],
1351 end_id
=int(parameters
["pushlog_id"]),
1355 for push_id
in sorted([int(p
) for p
in pushes
.keys()], reverse
=True):
1356 push_rev
= pushes
[str(push_id
)]["changesets"][-1]
1357 push_info
= find_hg_revision_push_info(
1358 "https://hg.mozilla.org/integration/" + parameters
["project"], push_rev
1360 pushdate
= int(push_info
["pushdate"])
1361 if date_is_yesterday(pushdate
) and reviewed_by_perftest(push_rev
):
1362 from gecko_taskgraph
.actions
.util
import trigger_action
1365 f
"Revision {push_rev} was created yesterday and was reviewed by "
1366 f
"#perftest-reviewers."
1369 push_decision_task_id
= get_decision_task_id(
1370 parameters
["project"], push_id
1373 print(f
"Could not find decision task for push {push_id}")
1377 action_name
="backfill-all-browsertime",
1378 # This lets the action know on which push we want to add a new task
1379 decision_task_id
=push_decision_task_id
,
1381 except Exception as e
:
1382 print(f
"Failed to trigger action for {push_rev}: {e}")
1387 @_target_task("condprof")
1388 def target_tasks_condprof(full_task_graph
, parameters
, graph_config
):
1390 Select tasks required for building conditioned profiles.
1392 for name
, task
in full_task_graph
.tasks
.items():
1393 if task
.kind
== "condprof":
1394 if "a51" not in name
: # bug 1765348
1398 @_target_task("system_symbols")
1399 def target_tasks_system_symbols(full_task_graph
, parameters
, graph_config
):
1401 Select tasks for scraping and uploading system symbols.
1403 for name
, task
in full_task_graph
.tasks
.items():
1406 "system-symbols-upload",
1407 "system-symbols-reprocess",
1412 @_target_task("perftest")
1413 def target_tasks_perftest(full_task_graph
, parameters
, graph_config
):
1415 Select perftest tasks we want to run daily
1417 for name
, task
in full_task_graph
.tasks
.items():
1418 if task
.kind
!= "perftest":
1420 if task
.attributes
.get("cron", False):
1424 @_target_task("perftest-on-autoland")
1425 def target_tasks_perftest_autoland(full_task_graph
, parameters
, graph_config
):
1427 Select perftest tasks we want to run daily
1429 for name
, task
in full_task_graph
.tasks
.items():
1430 if task
.kind
!= "perftest":
1432 if task
.attributes
.get("cron", False) and any(
1433 test_name
in name
for test_name
in ["view"]
1438 @_target_task("l10n-cross-channel")
1439 def target_tasks_l10n_cross_channel(full_task_graph
, parameters
, graph_config
):
1440 """Select the set of tasks required to run l10n cross-channel."""
1443 return task
.kind
in ["l10n-cross-channel"]
1445 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1448 @_target_task("are-we-esmified-yet")
1449 def target_tasks_are_we_esmified_yet(full_task_graph
, parameters
, graph_config
):
1451 select the task to track the progress of the esmification project
1454 l
for l
, t
in full_task_graph
.tasks
.items() if t
.kind
== "are-we-esmified-yet"
1458 @_target_task("eslint-build")
1459 def target_tasks_eslint_build(full_task_graph
, parameters
, graph_config
):
1460 """Select the task to run additional ESLint rules which require a build."""
1462 for name
, task
in full_task_graph
.tasks
.items():
1463 if task
.kind
!= "source-test":
1465 if "eslint-build" in name
:
1469 @_target_task("holly_tasks")
1470 def target_tasks_holly(full_task_graph
, parameters
, graph_config
):
1471 """Bug 1814661: only run updatebot tasks on holly"""
1474 return task
.kind
== "updatebot"
1476 return [l
for l
, t
in full_task_graph
.tasks
.items() if filter(t
)]
1479 @_target_task("snap_upstream_tests")
1480 def target_tasks_snap_upstream_tests(full_task_graph
, parameters
, graph_config
):
1482 Select tasks for testing Snap package built as upstream. Omit -try because
1483 it does not really make sense on a m-c cron
1485 for name
, task
in full_task_graph
.tasks
.items():
1486 if "snap-upstream-test" in name
and not "-try" in name
: