.taskcluster.yml

   1 # This file is rendered via JSON-e by
   2 # - mozilla-taskcluster - https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
   3 # - cron tasks - taskcluster/taskgraph/cron/decision.py
   4 # - action tasks - taskcluster/taskgraph/actions/registry.py
   5 version: 1
   6 tasks:
   7   # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
   8   # takes place.
   9   - $if: 'tasks_for in ["hg-push", "action", "cron"]'
  10     then:
  11       $let:
  12         # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
  13         ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
  14         # ensure there's no trailing `/` on the repo URL
  15         repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
  16         # Hardcode cron push info for now, so that we can transition to using real values without breaking callers of Chain of Trust
  17         _pushId: {$if: 'tasks_for == "cron"', then: '-1', else: {$eval: 'push.pushlog_id'}}
  18         # action tasks can fail because of no pushdate or push comment information in context, so include them in
  19         # hardcodes (even though they don't use these variables)
  20         _pushDate: {$if: 'tasks_for == "cron" || tasks_for == "action"', then: '0', else: {$eval: 'push.pushdate'}}
  21         _pushComment: {$if: 'tasks_for == "cron" || tasks_for == "action"', then: '', else: {$eval: 'push.comment'}}
  22       in:
  23         taskId: {$if: 'tasks_for != "action"', then: '${as_slugid("decision")}'}
  24         taskGroupId:
  25           $if: 'tasks_for == "action"'
  26           then:
  27             '${action.taskGroupId}'
  28           else:
  29             '${as_slugid("decision")}' # same as taskId; this is how automation identifies a decision tsak
  30         schedulerId: 'gecko-level-${repository.level}'
  31
  32         created: {$fromNow: ''}
  33         deadline: {$fromNow: '1 day'}
  34         expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
  35         metadata:
  36           $merge:
  37             - owner: "${ownerEmail}"
  38               source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
  39             - $if: 'tasks_for == "hg-push"'
  40               then:
  41                 name: "Gecko Decision Task"
  42                 description: 'The task that creates all of the other tasks in the task graph'
  43               else:
  44                 $if: 'tasks_for == "action"'
  45                 then:
  46                   name: "Action: ${action.title}"
  47                   description: '${action.description}'
  48                 else:
  49                   name: "Decision Task for cron job ${cron.job_name}"
  50                   description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
  51
  52         provisionerId: "aws-provisioner-v1"
  53         workerType: "gecko-${repository.level}-decision"
  54
  55         tags:
  56           $if: 'tasks_for == "hg-push"'
  57           then:
  58             createdForUser: "${ownerEmail}"
  59             kind: decision-task
  60           else:
  61             $if: 'tasks_for == "action"'
  62             then:
  63               createdForUser: '${ownerEmail}'
  64               kind: 'action-callback'
  65             else:
  66               $if: 'tasks_for == "cron"'
  67               then:
  68                 kind: cron-task
  69
  70         routes:
  71           $flatten:
  72             - "tc-treeherder.v2.${repository.project}.${push.revision}.${_pushId}"
  73             - $if: 'tasks_for == "hg-push"'
  74               then:
  75                 - "index.gecko.v2.${repository.project}.latest.taskgraph.decision"
  76                 - "index.gecko.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
  77                 - "index.gecko.v2.${repository.project}.pushlog-id.${_pushId}.decision"
  78                 - "notify.email.${ownerEmail}.on-failed"
  79                 - "notify.email.${ownerEmail}.on-exception"
  80                 # These are the old index routes for the decision task.
  81                 # They are still here so external tools that referenced them continue to work.
  82                 - "index.gecko.v2.${repository.project}.latest.firefox.decision"
  83                 - "index.gecko.v2.${repository.project}.revision.${push.revision}.firefox.decision"
  84               else:
  85                 $if: 'tasks_for == "action"'
  86                 then:
  87                 - "notify.email.taskcluster-notifications+action-task@mozilla.com.on-failed"
  88                 - "notify.email.taskcluster-notifications+action-task@mozilla.com.on-exception"
  89                 - "index.gecko.v2.${repository.project}.pushlog-id.${_pushId}.actions.${ownTaskId}"
  90                 else:  # cron
  91                 - "index.gecko.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
  92                 # These are the old index routes for the decision task.
  93                 - "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
  94
  95         scopes:
  96           $if: 'tasks_for == "hg-push"'
  97           then:
  98             - 'assume:repo:${repoUrl[8:]}:branch:default'
  99             - 'queue:route:notify.email.${ownerEmail}.*'
 100             - 'in-tree:hook-action:project-gecko/in-tree-action-${repository.level}-*'
 101           else:
 102             $if: 'tasks_for == "action"'
 103             then:
 104               # when all actions are hooks, we can calculate this directly rather than using a variable
 105               - '${action.repo_scope}'
 106             else:
 107               - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
 108
 109         dependencies: []
 110         requires: all-completed
 111
 112         priority:
 113           # Most times, there is plenty of worker capacity so everything runs
 114           # quickly, but sometimes a storm of action tasks lands.  Then we
 115           # want, from highest to lowest:
 116           # - cron tasks (time-sensitive) (low)
 117           # - decision tasks (minimize user-visible delay) (very-low)
 118           # - action tasks (avoid interfering with the other two) (lowest)
 119           # SCM levels all use different workerTypes, so there is no need for priority
 120           # between levels; "low" is the highest priority available at all levels, and
 121           # nothing runs at any higher priority on these workerTypes.
 122           $if: "tasks_for == 'cron'"
 123           then: low
 124           else:
 125             $if: "tasks_for == 'hg-push'"
 126             then: very-low
 127             else: lowest  # tasks_for == 'action'
 128         retries: 5
 129
 130         payload:
 131           env:
 132             # checkout-gecko uses these to check out the source; the inputs
 133             # to `mach taskgraph decision` are all on the command line.
 134             $merge:
 135               - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
 136                 GECKO_HEAD_REPOSITORY: '${repoUrl}'
 137                 GECKO_HEAD_REF: '${push.revision}'
 138                 GECKO_HEAD_REV: '${push.revision}'
 139                 GECKO_COMMIT_MSG: {$if: 'tasks_for != "action"', then: '${_pushComment}'}
 140                 HG_STORE_PATH: /builds/worker/checkouts/hg-store
 141                 TASKCLUSTER_CACHES: /builds/worker/checkouts
 142               - $if: 'tasks_for == "action"'
 143                 then:
 144                   ACTION_TASK_GROUP_ID: '${action.taskGroupId}'     # taskGroupId of the target task
 145                   ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
 146                   ACTION_INPUT: {$json: {$eval: 'input'}}
 147                   ACTION_CALLBACK: '${action.cb_name}'
 148                   ACTION_PARAMETERS: {$json: {$eval: 'parameters'}}
 149
 150           cache:
 151             level-${repository.level}-checkouts-sparse-v2: /builds/worker/checkouts
 152
 153           features:
 154             taskclusterProxy: true
 155             chainOfTrust: true
 156
 157           # Note: This task is built server side without the context or tooling that
 158           # exist in tree so we must hard code the hash
 159           image: 'taskcluster/decision:2.1.0@sha256:6db3b697d7a3c7aba440d72f04199331b872111cefff57206b8b8b1d53230360'
 160
 161           maxRunTime: 1800
 162
 163           command:
 164             - /builds/worker/bin/run-task
 165             - '--vcs-checkout=/builds/worker/checkouts/gecko'
 166             - '--sparse-profile=build/sparse-profiles/taskgraph'
 167             - '--'
 168             - bash
 169             - -cx
 170             - $let:
 171                 extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
 172               in:
 173                 $if: 'tasks_for == "action"'
 174                 then: >
 175                   cd /builds/worker/checkouts/gecko &&
 176                   ln -s /builds/worker/artifacts artifacts &&
 177                   ./mach --log-no-times taskgraph action-callback
 178                 else: >
 179                   cd /builds/worker/checkouts/gecko &&
 180                   ln -s /builds/worker/artifacts artifacts &&
 181                   ./mach --log-no-times taskgraph decision
 182                   --pushlog-id='${_pushId}'
 183                   --pushdate='${_pushDate}'
 184                   --project='${repository.project}'
 185                   --message="$GECKO_COMMIT_MSG"
 186                   --owner='${ownerEmail}'
 187                   --level='${repository.level}'
 188                   --base-repository="$GECKO_BASE_REPOSITORY"
 189                   --head-repository="$GECKO_HEAD_REPOSITORY"
 190                   --head-ref="$GECKO_HEAD_REF"
 191                   --head-rev="$GECKO_HEAD_REV"
 192                   ${extraArgs}
 193
 194           artifacts:
 195             'public':
 196               type: 'directory'
 197               path: '/builds/worker/artifacts'
 198               expires: {$fromNow: '1 year'}
 199
 200         extra:
 201           $merge:
 202             - treeherder:
 203                 $merge:
 204                   - machine:
 205                       platform: gecko-decision
 206                   - $if: 'tasks_for == "hg-push"'
 207                     then:
 208                       symbol: D
 209                     else:
 210                       $if: 'tasks_for == "action"'
 211                       then:
 212                         groupName: 'action-callback'
 213                         groupSymbol: AC
 214                         symbol: "${action.symbol}"
 215                       else:
 216                         groupSymbol: cron
 217                         symbol: "${cron.job_symbol}"
 218             - $if: 'tasks_for == "action"'
 219               then:
 220                 parent: '${action.taskGroupId}'
 221                 action:
 222                   name: '${action.name}'
 223                   context:
 224                     taskGroupId: '${action.taskGroupId}'
 225                     taskId: {$eval: 'taskId'}
 226                     input: {$eval: 'input'}
 227                     parameters: {$eval: 'parameters'}
 228             - $if: 'tasks_for == "cron"'
 229               then:
 230                 cron: {$json: {$eval: 'cron'}}
 231             - tasks_for: '${tasks_for}'