taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4 """
   5 Transform the beetmover task into an actual task description.
   6 """
   7
   8 import logging
   9
  10 from taskgraph.transforms.base import TransformSequence
  11 from taskgraph.util.dependencies import get_primary_dependency
  12 from taskgraph.util.schema import Schema
  13 from taskgraph.util.taskcluster import get_artifact_prefix
  14 from voluptuous import Optional, Required
  15
  16 from gecko_taskgraph.transforms.beetmover import craft_release_properties
  17 from gecko_taskgraph.transforms.task import task_description_schema
  18 from gecko_taskgraph.util.attributes import (
  19     copy_attributes_from_dependent_job,
  20 )
  21 from gecko_taskgraph.util.partners import get_ftp_platform, get_partner_config_by_kind
  22 from gecko_taskgraph.util.scriptworker import (
  23     add_scope_prefix,
  24     get_beetmover_bucket_scope,
  25 )
  26
  27 logger = logging.getLogger(__name__)
  28
  29
  30 beetmover_description_schema = Schema(
  31     {
  32         # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
  33         Optional("label"): str,
  34         Required("partner-path"): str,
  35         Optional("extra"): object,
  36         Optional("attributes"): task_description_schema["attributes"],
  37         Optional("dependencies"): task_description_schema["dependencies"],
  38         Required("shipping-phase"): task_description_schema["shipping-phase"],
  39         Optional("shipping-product"): task_description_schema["shipping-product"],
  40         Optional("priority"): task_description_schema["priority"],
  41         Optional("job-from"): task_description_schema["job-from"],
  42     }
  43 )
  44
  45 transforms = TransformSequence()
  46
  47
  48 @transforms.add
  49 def remove_name(config, jobs):
  50     for job in jobs:
  51         if "name" in job:
  52             del job["name"]
  53         yield job
  54
  55
  56 transforms.add_validate(beetmover_description_schema)
  57
  58
  59 @transforms.add
  60 def make_task_description(config, jobs):
  61     for job in jobs:
  62         dep_job = get_primary_dependency(config, job)
  63         assert dep_job
  64
  65         repack_id = dep_job.task.get("extra", {}).get("repack_id")
  66         if not repack_id:
  67             raise Exception("Cannot find repack id!")
  68
  69         attributes = dep_job.attributes
  70         build_platform = attributes.get("build_platform")
  71         if not build_platform:
  72             raise Exception("Cannot find build platform!")
  73
  74         label = dep_job.label.replace("repackage-signing-l10n", "beetmover-")
  75         label = dep_job.label.replace("repackage-signing-", "beetmover-")
  76         label = label.replace("repackage-", "beetmover-")
  77         label = label.replace("chunking-dummy-", "beetmover-")
  78         description = (
  79             "Beetmover submission for repack_id '{repack_id}' for build '"
  80             "{build_platform}/{build_type}'".format(
  81                 repack_id=repack_id,
  82                 build_platform=build_platform,
  83                 build_type=attributes.get("build_type"),
  84             )
  85         )
  86
  87         dependencies = {}
  88
  89         base_label = "release-partner-repack"
  90         if "eme" in config.kind:
  91             base_label = "release-eme-free-repack"
  92         dependencies["build"] = f"{base_label}-{build_platform}"
  93         if "macosx" in build_platform or "win" in build_platform:
  94             dependencies["repackage"] = "{}-repackage-{}-{}".format(
  95                 base_label, build_platform, repack_id.replace("/", "-")
  96             )
  97         dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format(
  98             base_label, build_platform, repack_id.replace("/", "-")
  99         )
 100
 101         attributes = copy_attributes_from_dependent_job(dep_job)
 102
 103         task = {
 104             "label": label,
 105             "description": description,
 106             "dependencies": dependencies,
 107             "attributes": attributes,
 108             "run-on-projects": dep_job.attributes.get("run_on_projects"),
 109             "shipping-phase": job["shipping-phase"],
 110             "shipping-product": job.get("shipping-product"),
 111             "partner-path": job["partner-path"],
 112             "extra": {
 113                 "repack_id": repack_id,
 114             },
 115         }
 116         # we may have reduced the priority for partner jobs, otherwise task.py will set it
 117         if job.get("priority"):
 118             task["priority"] = job["priority"]
 119
 120         yield task
 121
 122
 123 @transforms.add
 124 def populate_scopes_and_worker_type(config, jobs):
 125     bucket_scope = get_beetmover_bucket_scope(config)
 126     action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner")
 127
 128     for job in jobs:
 129         job["scopes"] = [bucket_scope, action_scope]
 130         job["worker-type"] = "beetmover"
 131         yield job
 132
 133
 134 def generate_upstream_artifacts(
 135     job,
 136     build_task_ref,
 137     repackage_task_ref,
 138     repackage_signing_task_ref,
 139     platform,
 140     repack_id,
 141     partner_path,
 142     repack_stub_installer=False,
 143 ):
 144     upstream_artifacts = []
 145     artifact_prefix = get_artifact_prefix(job)
 146
 147     if "linux" in platform:
 148         upstream_artifacts.append(
 149             {
 150                 "taskId": {"task-reference": build_task_ref},
 151                 "taskType": "build",
 152                 "paths": [f"{artifact_prefix}/{repack_id}/target.tar.bz2"],
 153                 "locale": partner_path,
 154             }
 155         )
 156         upstream_artifacts.append(
 157             {
 158                 "taskId": {"task-reference": repackage_signing_task_ref},
 159                 "taskType": "repackage",
 160                 "paths": [f"{artifact_prefix}/{repack_id}/target.tar.bz2.asc"],
 161                 "locale": partner_path,
 162             }
 163         )
 164     elif "macosx" in platform:
 165         upstream_artifacts.append(
 166             {
 167                 "taskId": {"task-reference": repackage_task_ref},
 168                 "taskType": "repackage",
 169                 "paths": [f"{artifact_prefix}/{repack_id}/target.dmg"],
 170                 "locale": partner_path,
 171             }
 172         )
 173         upstream_artifacts.append(
 174             {
 175                 "taskId": {"task-reference": repackage_signing_task_ref},
 176                 "taskType": "repackage",
 177                 "paths": [f"{artifact_prefix}/{repack_id}/target.dmg.asc"],
 178                 "locale": partner_path,
 179             }
 180         )
 181     elif "win" in platform:
 182         upstream_artifacts.append(
 183             {
 184                 "taskId": {"task-reference": repackage_signing_task_ref},
 185                 "taskType": "repackage",
 186                 "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe"],
 187                 "locale": partner_path,
 188             }
 189         )
 190         upstream_artifacts.append(
 191             {
 192                 "taskId": {"task-reference": repackage_signing_task_ref},
 193                 "taskType": "repackage",
 194                 "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe.asc"],
 195                 "locale": partner_path,
 196             }
 197         )
 198         if platform.startswith("win32") and repack_stub_installer:
 199             upstream_artifacts.append(
 200                 {
 201                     "taskId": {"task-reference": repackage_signing_task_ref},
 202                     "taskType": "repackage",
 203                     "paths": [
 204                         "{}/{}/target.stub-installer.exe".format(
 205                             artifact_prefix, repack_id
 206                         )
 207                     ],
 208                     "locale": partner_path,
 209                 }
 210             )
 211             upstream_artifacts.append(
 212                 {
 213                     "taskId": {"task-reference": repackage_signing_task_ref},
 214                     "taskType": "repackage",
 215                     "paths": [
 216                         "{}/{}/target.stub-installer.exe.asc".format(
 217                             artifact_prefix, repack_id
 218                         )
 219                     ],
 220                     "locale": partner_path,
 221                 }
 222             )
 223
 224     if not upstream_artifacts:
 225         raise Exception("Couldn't find any upstream artifacts.")
 226
 227     return upstream_artifacts
 228
 229
 230 @transforms.add
 231 def make_task_worker(config, jobs):
 232     for job in jobs:
 233         platform = job["attributes"]["build_platform"]
 234         repack_id = job["extra"]["repack_id"]
 235         partner, subpartner, locale = job["extra"]["repack_id"].split("/")
 236         partner_config = get_partner_config_by_kind(config, config.kind)
 237         repack_stub_installer = partner_config[partner][subpartner].get(
 238             "repack_stub_installer"
 239         )
 240         build_task = None
 241         repackage_task = None
 242         repackage_signing_task = None
 243
 244         for dependency in job["dependencies"].keys():
 245             if "repackage-signing" in dependency:
 246                 repackage_signing_task = dependency
 247             elif "repackage" in dependency:
 248                 repackage_task = dependency
 249             else:
 250                 build_task = "build"
 251
 252         build_task_ref = "<" + str(build_task) + ">"
 253         repackage_task_ref = "<" + str(repackage_task) + ">"
 254         repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"
 255
 256         # generate the partner path; we'll send this to beetmover as the "locale"
 257         ftp_platform = get_ftp_platform(platform)
 258         repl_dict = {
 259             "build_number": config.params["build_number"],
 260             "locale": locale,
 261             "partner": partner,
 262             "platform": ftp_platform,
 263             "release_partner_build_number": config.params[
 264                 "release_partner_build_number"
 265             ],
 266             "subpartner": subpartner,
 267             "version": config.params["version"],
 268         }
 269         partner_path = job["partner-path"].format(**repl_dict)
 270         del job["partner-path"]
 271
 272         worker = {
 273             "implementation": "beetmover",
 274             "release-properties": craft_release_properties(config, job),
 275             "upstream-artifacts": generate_upstream_artifacts(
 276                 job,
 277                 build_task_ref,
 278                 repackage_task_ref,
 279                 repackage_signing_task_ref,
 280                 platform,
 281                 repack_id,
 282                 partner_path,
 283                 repack_stub_installer,
 284             ),
 285         }
 286         job["worker"] = worker
 287
 288         yield job