no bug - Bumping Firefox l10n changesets r=release a=l10n-bump DONTBUILD CLOSED TREE

[gecko.git] / taskcluster / gecko_taskgraph / transforms / repackage.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the repackage task into an actual task description.
"""

from taskgraph.transforms.base import TransformSequence
from taskgraph.util.dependencies import get_primary_dependency
from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
from taskgraph.util.taskcluster import get_artifact_prefix
from voluptuous import Extra, Optional, Required

from gecko_taskgraph.transforms.job import job_description_schema
from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
from gecko_taskgraph.util.copy_task import copy_task
from gecko_taskgraph.util.platforms import architecture, archive_format
from gecko_taskgraph.util.workertypes import worker_type_implementation

packaging_description_schema = Schema(
    {
        # unique label to describe this repackaging task
        Optional("label"): str,
        Optional("worker-type"): str,
        Optional("worker"): object,
        Optional("attributes"): job_description_schema["attributes"],
        Optional("dependencies"): job_description_schema["dependencies"],
        # treeherder is allowed here to override any defaults we use for repackaging.  See
        # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the
        # below transforms for defaults of various values.
        Optional("treeherder"): job_description_schema["treeherder"],
        # If a l10n task, the corresponding locale
        Optional("locale"): str,
        # Routes specific to this task, if defined
        Optional("routes"): [str],
        # passed through directly to the job description
        Optional("extra"): job_description_schema["extra"],
        # passed through to job description
        Optional("fetches"): job_description_schema["fetches"],
        Optional("run-on-projects"): job_description_schema["run-on-projects"],
        # Shipping product and phase
        Optional("shipping-product"): job_description_schema["shipping-product"],
        Optional("shipping-phase"): job_description_schema["shipping-phase"],
        Required("package-formats"): optionally_keyed_by(
            "build-platform", "release-type", "build-type", [str]
        ),
        Optional("msix"): {
            Optional("channel"): optionally_keyed_by(
                "package-format",
                "level",
                "build-platform",
                "release-type",
                "shipping-product",
                str,
            ),
            Optional("identity-name"): optionally_keyed_by(
                "package-format",
                "level",
                "build-platform",
                "release-type",
                "shipping-product",
                str,
            ),
            Optional("publisher"): optionally_keyed_by(
                "package-format",
                "level",
                "build-platform",
                "release-type",
                "shipping-product",
                str,
            ),
            Optional("publisher-display-name"): optionally_keyed_by(
                "package-format",
                "level",
                "build-platform",
                "release-type",
                "shipping-product",
                str,
            ),
            Optional("vendor"): str,
        },
        # All l10n jobs use mozharness
        Required("mozharness"): {
            Extra: object,
            # Config files passed to the mozharness script
            Required("config"): optionally_keyed_by("build-platform", [str]),
            # Additional paths to look for mozharness configs in. These should be
            # relative to the base of the source checkout
            Optional("config-paths"): [str],
            # if true, perform a checkout of a comm-central based branch inside the
            # gecko checkout
            Optional("comm-checkout"): bool,
            Optional("run-as-root"): bool,
            Optional("use-caches"): bool,
        },
        Optional("job-from"): job_description_schema["job-from"],
    }
)

# The configuration passed to the mozharness repackage script. This defines the
# arguments passed to `mach repackage`
# - `args` is interpolated by mozharness (`{package-name}`, `{installer-tag}`,
#   `{stub-installer-tag}`, `{sfx-stub}`, `{wsx-stub}`, `{fetch-dir}`), with values
#    from mozharness.
# - `inputs` are passed as long-options, with the filename prefixed by
#   `MOZ_FETCH_DIR`. The filename is interpolated by taskgraph
#   (`{archive_format}`).
# - `output` is passed to `--output`, with the filename prefixed by the output
#   directory.
PACKAGE_FORMATS = {
    "mar": {
        "args": [
            "mar",
            "--arch",
            "{architecture}",
            "--mar-channel-id",
            "{mar-channel-id}",
        ],
        "inputs": {
            "input": "target{archive_format}",
            "mar": "mar-tools/mar",
        },
        "output": "target.complete.mar",
    },
    "msi": {
        "args": [
            "msi",
            "--wsx",
            "{wsx-stub}",
            "--version",
            "{version_display}",
            "--locale",
            "{_locale}",
            "--arch",
            "{architecture}",
            "--candle",
            "{fetch-dir}/candle.exe",
            "--light",
            "{fetch-dir}/light.exe",
        ],
        "inputs": {
            "setupexe": "target.installer.exe",
        },
        "output": "target.installer.msi",
    },
    "msix": {
        "args": [
            "msix",
            "--channel",
            "{msix-channel}",
            "--publisher",
            "{msix-publisher}",
            "--publisher-display-name",
            "{msix-publisher-display-name}",
            "--identity-name",
            "{msix-identity-name}",
            "--vendor",
            "{msix-vendor}",
            "--arch",
            "{architecture}",
            # For langpacks.  Ignored if directory does not exist.
            "--distribution-dir",
            "{fetch-dir}/distribution",
            "--verbose",
            "--makeappx",
            "{fetch-dir}/msix-packaging/makemsix",
        ],
        "inputs": {
            "input": "target{archive_format}",
        },
        "output": "target.installer.msix",
    },
    "msix-store": {
        "args": [
            "msix",
            "--channel",
            "{msix-channel}",
            "--publisher",
            "{msix-publisher}",
            "--publisher-display-name",
            "{msix-publisher-display-name}",
            "--identity-name",
            "{msix-identity-name}",
            "--vendor",
            "{msix-vendor}",
            "--arch",
            "{architecture}",
            # For langpacks.  Ignored if directory does not exist.
            "--distribution-dir",
            "{fetch-dir}/distribution",
            "--verbose",
            "--makeappx",
            "{fetch-dir}/msix-packaging/makemsix",
        ],
        "inputs": {
            "input": "target{archive_format}",
        },
        "output": "target.store.msix",
    },
    "dmg": {
        "args": ["dmg"],
        "inputs": {
            "input": "target{archive_format}",
        },
        "output": "target.dmg",
    },
    "dmg-attrib": {
        "args": [
            "dmg",
            "--attribution_sentinel",
            "__MOZCUSTOM__",
        ],
        "inputs": {
            "input": "target{archive_format}",
        },
        "output": "target.dmg",
    },
    "pkg": {
        "args": ["pkg"],
        "inputs": {
            "input": "target{archive_format}",
        },
        "output": "target.pkg",
    },
    "installer": {
        "args": [
            "installer",
            "--package-name",
            "{package-name}",
            "--tag",
            "{installer-tag}",
            "--sfx-stub",
            "{sfx-stub}",
        ],
        "inputs": {
            "package": "target{archive_format}",
            "setupexe": "setup.exe",
        },
        "output": "target.installer.exe",
    },
    "installer-stub": {
        "args": [
            "installer",
            "--tag",
            "{stub-installer-tag}",
            "--sfx-stub",
            "{sfx-stub}",
        ],
        "inputs": {
            "setupexe": "setup-stub.exe",
        },
        "output": "target.stub-installer.exe",
    },
    "deb": {
        "args": [
            "deb",
            "--arch",
            "{architecture}",
            "--templates",
            "browser/installer/linux/app/debian",
            "--version",
            "{version_display}",
            "--build-number",
            "{build_number}",
            "--release-product",
            "{release_product}",
            "--release-type",
            "{release_type}",
        ],
        "inputs": {
            "input": "target{archive_format}",
        },
        "output": "target.deb",
    },
    "deb-l10n": {
        "args": [
            "deb-l10n",
            "--version",
            "{version_display}",
            "--build-number",
            "{build_number}",
            "--templates",
            "browser/installer/linux/langpack/debian",
            "--release-product",
            "{release_product}",
        ],
        "inputs": {
            "input-xpi-file": "target.langpack.xpi",
            "input-tar-file": "target{archive_format}",
        },
        "output": "target.langpack.deb",
    },
}
MOZHARNESS_EXPANSIONS = [
    "package-name",
    "installer-tag",
    "fetch-dir",
    "stub-installer-tag",
    "sfx-stub",
    "wsx-stub",
]

transforms = TransformSequence()


@transforms.add
def remove_name(config, jobs):
    for job in jobs:
        if "name" in job:
            del job["name"]
        yield job


transforms.add_validate(packaging_description_schema)


@transforms.add
def copy_in_useful_magic(config, jobs):
    """Copy attributes from upstream task to be used for keyed configuration."""
    for job in jobs:
        dep = get_primary_dependency(config, job)
        assert dep

        job["build-platform"] = dep.attributes.get("build_platform")
        job["shipping-product"] = dep.attributes.get("shipping_product")
        job["build-type"] = dep.attributes.get("build_type")
        yield job


@transforms.add
def handle_keyed_by(config, jobs):
    """Resolve fields that can be keyed by platform, etc, but not `msix.*` fields
    that can be keyed by `package-format`.  Such fields are handled specially below.
    """
    fields = [
        "mozharness.config",
        "package-formats",
        "worker.max-run-time",
    ]
    for job in jobs:
        job = copy_task(job)  # don't overwrite dict values here
        for field in fields:
            resolve_keyed_by(
                item=job,
                field=field,
                item_name="?",
                **{
                    "release-type": config.params["release_type"],
                    "level": config.params["level"],
                },
            )
        yield job


@transforms.add
def make_repackage_description(config, jobs):
    for job in jobs:
        dep_job = get_primary_dependency(config, job)
        assert dep_job

        label = job.get("label", dep_job.label.replace("signing-", "repackage-"))
        job["label"] = label

        yield job


@transforms.add
def make_job_description(config, jobs):
    for job in jobs:
        dep_job = get_primary_dependency(config, job)
        assert dep_job

        dependencies = {dep_job.kind: dep_job.label}

        attributes = copy_attributes_from_dependent_job(dep_job)
        attributes["repackage_type"] = "repackage"

        locale = attributes.get("locale", job.get("locale"))
        if locale:
            attributes["locale"] = locale

        description = (
            "Repackaging for locale '{locale}' for build '"
            "{build_platform}/{build_type}'".format(
                locale=attributes.get("locale", "en-US"),
                build_platform=attributes.get("build_platform"),
                build_type=attributes.get("build_type"),
            )
        )

        treeherder = job.get("treeherder", {})
        treeherder.setdefault("symbol", "Rpk")
        dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform")
        treeherder.setdefault("platform", dep_th_platform)
        treeherder.setdefault("tier", 1)
        treeherder.setdefault("kind", "build")

        # Search dependencies before adding langpack dependencies.
        signing_task = None
        repackage_signing_task = None
        for dependency in dependencies.keys():
            if "repackage-signing" in dependency:
                repackage_signing_task = dependency
            elif "signing" in dependency or "notarization" in dependency:
                signing_task = dependency

        if config.kind == "repackage-msi":
            treeherder["symbol"] = "MSI({})".format(locale or "N")

        elif config.kind == "repackage-msix":
            assert not locale

            # Like "MSIXs(Bs)".
            treeherder["symbol"] = "MSIX({})".format(
                dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B")
            )

        elif config.kind == "repackage-shippable-l10n-msix":
            assert not locale

            if attributes.get("l10n_chunk") or attributes.get("chunk_locales"):
                # We don't want to produce MSIXes for single-locale repack builds.
                continue

            description = (
                "Repackaging with multiple locales for build '"
                "{build_platform}/{build_type}'".format(
                    build_platform=attributes.get("build_platform"),
                    build_type=attributes.get("build_type"),
                )
            )

            # Like "MSIXs(Bs-multi)".
            treeherder["symbol"] = "MSIX({}-multi)".format(
                dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B")
            )

            fetches = job.setdefault("fetches", {})

            # The keys are unique, like `shippable-l10n-signing-linux64-shippable-1/opt`, so we
            # can't ask for the tasks directly, we must filter for them.
            for t in config.kind_dependencies_tasks.values():
                if t.kind != "shippable-l10n-signing":
                    continue
                if t.attributes["build_platform"] != "linux64-shippable":
                    continue
                if t.attributes["build_type"] != "opt":
                    continue

                dependencies.update({t.label: t.label})

                fetches.update(
                    {
                        t.label: [
                            {
                                "artifact": f"{loc}/target.langpack.xpi",
                                "extract": False,
                                # Otherwise we can't disambiguate locales!
                                "dest": f"distribution/extensions/{loc}",
                            }
                            for loc in t.attributes["chunk_locales"]
                        ]
                    }
                )

        elif config.kind == "repackage-deb":
            attributes["repackage_type"] = "repackage-deb"
            description = (
                "Repackaging the '{build_platform}/{build_type}' "
                "{version} build into a '.deb' package"
            ).format(
                build_platform=attributes.get("build_platform"),
                build_type=attributes.get("build_type"),
                version=config.params["version"],
            )

        _fetch_subst_locale = "en-US"
        if locale:
            _fetch_subst_locale = locale

        worker_type = job["worker-type"]
        build_platform = attributes["build_platform"]

        use_stub = attributes.get("stub-installer")

        repackage_config = []
        package_formats = job.get("package-formats")
        if use_stub and not repackage_signing_task and "msix" not in package_formats:
            # if repackage_signing_task doesn't exists, generate the stub installer
            package_formats += ["installer-stub"]
        for format in package_formats:
            command = copy_task(PACKAGE_FORMATS[format])
            substs = {
                "archive_format": archive_format(build_platform),
                "_locale": _fetch_subst_locale,
                "architecture": architecture(build_platform),
                "version_display": config.params["version"],
                "mar-channel-id": attributes["mar-channel-id"],
                "build_number": config.params["build_number"],
                "release_product": config.params["release_product"],
                "release_type": config.params["release_type"],
            }
            # Allow us to replace `args` as well, but specifying things expanded in mozharness
            # without breaking .format and without allowing unknown through.
            substs.update({name: f"{{{name}}}" for name in MOZHARNESS_EXPANSIONS})

            # We need to resolve `msix.*` values keyed by `package-format` for each format, not
            # just once, so we update a temporary copy just for extracting these values.
            temp_job = copy_task(job)
            for msix_key in (
                "channel",
                "identity-name",
                "publisher",
                "publisher-display-name",
                "vendor",
            ):
                resolve_keyed_by(
                    item=temp_job,
                    field=f"msix.{msix_key}",
                    item_name="?",
                    **{
                        "package-format": format,
                        "release-type": config.params["release_type"],
                        "level": config.params["level"],
                    },
                )

                # Turn `msix.channel` into `msix-channel`, etc.
                value = temp_job.get("msix", {}).get(msix_key)
                if value:
                    substs.update(
                        {f"msix-{msix_key}": value},
                    )

            command["inputs"] = {
                name: filename.format(**substs)
                for name, filename in command["inputs"].items()
            }
            command["args"] = [arg.format(**substs) for arg in command["args"]]
            if "installer" in format and "aarch64" not in build_platform:
                command["args"].append("--use-upx")

            repackage_config.append(command)

        run = job.get("mozharness", {})
        run.update(
            {
                "using": "mozharness",
                "script": "mozharness/scripts/repackage.py",
                "job-script": "taskcluster/scripts/builder/repackage.sh",
                "actions": ["setup", "repackage"],
                "extra-config": {
                    "repackage_config": repackage_config,
                },
                "run-as-root": run.get("run-as-root", False),
                "use-caches": run.get("use-caches", True),
            }
        )

        worker = job.get("worker", {})
        worker.update(
            {
                "chain-of-trust": True,
                # Don't add generic artifact directory.
                "skip-artifacts": True,
            }
        )
        worker.setdefault("max-run-time", 3600)

        if locale:
            # Make sure we specify the locale-specific upload dir
            worker.setdefault("env", {})["LOCALE"] = locale

        worker["artifacts"] = _generate_task_output_files(
            dep_job,
            worker_type_implementation(config.graph_config, config.params, worker_type),
            repackage_config=repackage_config,
            locale=locale,
        )
        attributes["release_artifacts"] = [
            artifact["name"] for artifact in worker["artifacts"]
        ]

        task = {
            "label": job["label"],
            "description": description,
            "worker-type": worker_type,
            "dependencies": dependencies,
            "if-dependencies": [dep_job.kind],
            "attributes": attributes,
            "run-on-projects": job.get(
                "run-on-projects", dep_job.attributes.get("run_on_projects")
            ),
            "optimization": dep_job.optimization,
            "treeherder": treeherder,
            "routes": job.get("routes", []),
            "extra": job.get("extra", {}),
            "worker": worker,
            "run": run,
            "fetches": _generate_download_config(
                config,
                dep_job,
                build_platform,
                signing_task,
                repackage_signing_task,
                locale=locale,
                existing_fetch=job.get("fetches"),
            ),
        }

        if build_platform.startswith("macosx"):
            task.setdefault("fetches", {}).setdefault("toolchain", []).extend(
                [
                    "linux64-libdmg",
                    "linux64-hfsplus",
                    "linux64-node",
                    "linux64-xar",
                    "linux64-mkbom",
                ]
            )

        if "shipping-phase" in job:
            task["shipping-phase"] = job["shipping-phase"]

        yield task


def _generate_download_config(
    config,
    task,
    build_platform,
    signing_task,
    repackage_signing_task,
    locale=None,
    existing_fetch=None,
):
    locale_path = f"{locale}/" if locale else ""
    fetch = {}
    if existing_fetch:
        fetch.update(existing_fetch)

    if repackage_signing_task and build_platform.startswith("win"):
        fetch.update(
            {
                repackage_signing_task: [f"{locale_path}target.installer.exe"],
            }
        )
    elif build_platform.startswith("linux") or build_platform.startswith("macosx"):
        signing_fetch = [
            {
                "artifact": "{}target{}".format(
                    locale_path, archive_format(build_platform)
                ),
                "extract": False,
            },
        ]
        if config.kind == "repackage-deb-l10n":
            signing_fetch.append(
                {
                    "artifact": f"{locale_path}target.langpack.xpi",
                    "extract": False,
                }
            )
        fetch.update({signing_task: signing_fetch})
    elif build_platform.startswith("win"):
        fetch.update(
            {
                signing_task: [
                    {
                        "artifact": f"{locale_path}target.zip",
                        "extract": False,
                    },
                    f"{locale_path}setup.exe",
                ],
            }
        )

        use_stub = task.attributes.get("stub-installer")
        if use_stub:
            fetch[signing_task].append(f"{locale_path}setup-stub.exe")

    if fetch:
        return fetch

    raise NotImplementedError(f'Unsupported build_platform: "{build_platform}"')


def _generate_task_output_files(
    task, worker_implementation, repackage_config, locale=None
):
    locale_output_path = f"{locale}/" if locale else ""
    artifact_prefix = get_artifact_prefix(task)

    if worker_implementation == ("docker-worker", "linux"):
        local_prefix = "/builds/worker/workspace/"
    elif worker_implementation == ("generic-worker", "windows"):
        local_prefix = "workspace/"
    else:
        raise NotImplementedError(
            f'Unsupported worker implementation: "{worker_implementation}"'
        )

    output_files = []
    for config in repackage_config:
        output_files.append(
            {
                "type": "file",
                "path": "{}outputs/{}{}".format(
                    local_prefix, locale_output_path, config["output"]
                ),
                "name": "{}/{}{}".format(
                    artifact_prefix, locale_output_path, config["output"]
                ),
            }
        )
    return output_files