taskcluster/android_taskgraph/job.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5
   6 from pipes import quote as shell_quote
   7
   8 from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using
   9 from taskgraph.util import path
  10 from taskgraph.util.schema import Schema, taskref_or_string
  11 from voluptuous import Optional, Required
  12
  13 secret_schema = {
  14     Required("name"): str,
  15     Required("path"): str,
  16     Required("key"): str,
  17     Optional("json"): bool,
  18     Optional("decode"): bool,
  19 }
  20
  21 dummy_secret_schema = {
  22     Required("content"): str,
  23     Required("path"): str,
  24     Optional("json"): bool,
  25 }
  26
  27 gradlew_schema = Schema(
  28     {
  29         Required("using"): "gradlew",
  30         Optional("pre-gradlew"): [[str]],
  31         Required("gradlew"): [str],
  32         Optional("post-gradlew"): [[str]],
  33         # Base work directory used to set up the task.
  34         Required("workdir"): str,
  35         Optional("use-caches"): bool,
  36         Optional("secrets"): [secret_schema],
  37         Optional("dummy-secrets"): [dummy_secret_schema],
  38     }
  39 )
  40
  41 run_commands_schema = Schema(
  42     {
  43         Required("using"): "run-commands",
  44         Optional("pre-commands"): [[str]],
  45         Required("commands"): [[taskref_or_string]],
  46         Required("workdir"): str,
  47         Optional("use-caches"): bool,
  48         Optional("secrets"): [secret_schema],
  49         Optional("dummy-secrets"): [dummy_secret_schema],
  50     }
  51 )
  52
  53
  54 @run_job_using("docker-worker", "run-commands", schema=run_commands_schema)
  55 def configure_run_commands_schema(config, job, taskdesc):
  56     run = job["run"]
  57     pre_commands = run.pop("pre-commands", [])
  58     pre_commands += [
  59         _generate_dummy_secret_command(secret)
  60         for secret in run.pop("dummy-secrets", [])
  61     ]
  62     pre_commands += [
  63         _generate_secret_command(secret) for secret in run.get("secrets", [])
  64     ]
  65
  66     all_commands = pre_commands + run.pop("commands", [])
  67
  68     run["command"] = _convert_commands_to_string(all_commands)
  69     _inject_secrets_scopes(run, taskdesc)
  70     _set_run_task_attributes(job)
  71     configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
  72
  73
  74 @run_job_using("docker-worker", "gradlew", schema=gradlew_schema)
  75 def configure_gradlew(config, job, taskdesc):
  76     run = job["run"]
  77     worker = taskdesc["worker"] = job["worker"]
  78
  79     fetches_dir = "/builds/worker/fetches"
  80     topsrc_dir = "/builds/worker/checkouts/gecko"
  81     worker.setdefault("env", {}).update(
  82         {
  83             "ANDROID_SDK_ROOT": path.join(fetches_dir, "android-sdk-linux"),
  84             "GRADLE_USER_HOME": path.join(
  85                 topsrc_dir, "mobile/android/gradle/dotgradle-online"
  86             ),
  87             "MOZ_BUILD_DATE": config.params["moz_build_date"],
  88         }
  89     )
  90     worker["env"].setdefault(
  91         "MOZCONFIG",
  92         path.join(
  93             topsrc_dir,
  94             "mobile/android/config/mozconfigs/android-arm/nightly-android-lints",
  95         ),
  96     )
  97     worker["env"].setdefault(
  98         "MOZ_ANDROID_FAT_AAR_ARCHITECTURES", "armeabi-v7a,arm64-v8a,x86,x86_64"
  99     )
 100
 101     dummy_secrets = [
 102         _generate_dummy_secret_command(secret)
 103         for secret in run.pop("dummy-secrets", [])
 104     ]
 105     secrets = [_generate_secret_command(secret) for secret in run.get("secrets", [])]
 106     worker["env"].update(
 107         {
 108             "PRE_GRADLEW": _convert_commands_to_string(run.pop("pre-gradlew", [])),
 109             "GET_SECRETS": _convert_commands_to_string(dummy_secrets + secrets),
 110             "GRADLEW_ARGS": " ".join(run.pop("gradlew")),
 111             "POST_GRADLEW": _convert_commands_to_string(run.pop("post-gradlew", [])),
 112         }
 113     )
 114     run[
 115         "command"
 116     ] = "/builds/worker/checkouts/gecko/taskcluster/scripts/builder/build-android.sh"
 117     _inject_secrets_scopes(run, taskdesc)
 118     _set_run_task_attributes(job)
 119     configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
 120
 121
 122 def _generate_secret_command(secret):
 123     secret_command = [
 124         "/builds/worker/checkouts/gecko/taskcluster/scripts/get-secret.py",
 125         "-s",
 126         secret["name"],
 127         "-k",
 128         secret["key"],
 129         "-f",
 130         secret["path"],
 131     ]
 132     if secret.get("json"):
 133         secret_command.append("--json")
 134
 135     if secret.get("decode"):
 136         secret_command.append("--decode")
 137
 138     return secret_command
 139
 140
 141 def _generate_dummy_secret_command(secret):
 142     secret_command = [
 143         "/builds/worker/checkouts/gecko/taskcluster/scripts/write-dummy-secret.py",
 144         "-f",
 145         secret["path"],
 146         "-c",
 147         secret["content"],
 148     ]
 149     if secret.get("json"):
 150         secret_command.append("--json")
 151
 152     return secret_command
 153
 154
 155 def _convert_commands_to_string(commands):
 156     should_artifact_reference = False
 157     should_task_reference = False
 158
 159     sanitized_commands = []
 160     for command in commands:
 161         sanitized_parts = []
 162         for part in command:
 163             if isinstance(part, dict):
 164                 if "artifact-reference" in part:
 165                     part_string = part["artifact-reference"]
 166                     should_artifact_reference = True
 167                 elif "task-reference" in part:
 168                     part_string = part["task-reference"]
 169                     should_task_reference = True
 170                 else:
 171                     raise ValueError(f"Unsupported dict: {part}")
 172             else:
 173                 part_string = part
 174
 175             sanitized_parts.append(part_string)
 176         sanitized_commands.append(sanitized_parts)
 177
 178     shell_quoted_commands = [
 179         " ".join(map(shell_quote, command)) for command in sanitized_commands
 180     ]
 181     full_string_command = " && ".join(shell_quoted_commands)
 182
 183     if should_artifact_reference and should_task_reference:
 184         raise NotImplementedError(
 185             '"arifact-reference" and "task-reference" cannot be both used'
 186         )
 187     elif should_artifact_reference:
 188         return {"artifact-reference": full_string_command}
 189     elif should_task_reference:
 190         return {"task-reference": full_string_command}
 191     else:
 192         return full_string_command
 193
 194
 195 def _inject_secrets_scopes(run, taskdesc):
 196     secrets = run.pop("secrets", [])
 197     scopes = taskdesc.setdefault("scopes", [])
 198     new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets]
 199     new_secret_scopes = list(
 200         set(new_secret_scopes)
 201     )  # Scopes must not have any duplicates
 202     scopes.extend(new_secret_scopes)
 203
 204
 205 def _set_run_task_attributes(job):
 206     run = job["run"]
 207     run["cwd"] = "{checkout}"
 208     run["using"] = "run-task"