.taskcluster.yml

   1 # This file is rendered via JSON-e by
   2 # - mozilla-taskcluster - See
   3 #   https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
   4 #   {
   5 #     tasks_for: 'hg-push',
   6 #     push: {owner, comment, pushlog_id, pushdate},
   7 #     repository: {url, project, level},
   8 #     now,
   9 #     as_slugid: // function
  10 #     ownTaskId: // taskId of the task that will be created
  11 #   }
  12 #
  13 # - cron tasks - See taskcluster/taskgraph/cron/decision.py
  14 #   {
  15 #     tasks_for: 'cron',
  16 #     push: {revision, pushlog_id, pushdate, owner}
  17 #     repository: {url, project, level},
  18 #     cron: {task_id, job_name, job_symbol, quoted_args},
  19 #     now,
  20 #     ownTaskId: // taskId of the task that will be created
  21 #   }
  22 #
  23 # - action tasks - See:
  24 #   * taskcluster/taskgraph/actions/registry.py,
  25 #   * https://docs.taskcluster.net/docs/manual/using/actions/spec
  26 #   * ci-admin:ciadmin/generate/in_tree_actions.py
  27 #
  28 #   The registry generates the hookPayload that appears in actions.json, and
  29 #   contains data from the decision task as well as JSON-e code to combine that
  30 #   with data supplied as part of the action spec.  When the hook is fired, the
  31 #   hookPayload is rendered with JSON-e to produce a payload for the hook task
  32 #   template.
  33 #
  34 #   The ci-admin code wraps the content of this file (.taskcluster.yml) with a
  35 #   JSON-e $let statement that produces the context described below, and
  36 #   installs that as the hook task template.
  37 #
  38 #   {
  39 #     tasks_for: 'action',
  40 #     push: {owner, pushlog_id, revision},
  41 #     repository: {url, project, level},
  42 #     input,
  43 #     taskId,      // targetted taskId
  44 #     taskGroupId, // targetted taskGroupId
  45 #     action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
  46 #     ownTaskId:   // taskId of the task that will be created
  47 #     clientId:    // clientId that triggered this hook
  48 #   }
  49
  50 version: 1
  51 tasks:
  52   # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
  53   # takes place.
  54   - $if: 'tasks_for in ["hg-push", "action", "cron"]'
  55     then:
  56       $let:
  57         # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
  58         ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
  59         # ensure there's no trailing `/` on the repo URL
  60         repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
  61         # expire try earlier than other branches
  62         expires:
  63           $if: 'repository.project == "try"'
  64           then: {$fromNow: '28 days'}
  65           else: {$fromNow: '1 year'}
  66       in:
  67         taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
  68         taskGroupId:
  69           $if: 'tasks_for == "action"'
  70           then:
  71             '${action.taskGroupId}'
  72           else:
  73             '${ownTaskId}' # same as taskId; this is how automation identifies a decision tsak
  74         schedulerId: 'gecko-level-${repository.level}'
  75
  76         created: {$fromNow: ''}
  77         deadline: {$fromNow: '1 day'}
  78         expires: {$eval: 'expires'}
  79         metadata:
  80           $merge:
  81             - owner: "${ownerEmail}"
  82               source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
  83             - $if: 'tasks_for == "hg-push"'
  84               then:
  85                 name: "Gecko Decision Task"
  86                 description: 'The task that creates all of the other tasks in the task graph'
  87               else:
  88                 $if: 'tasks_for == "action"'
  89                 then:
  90                   name: "Action: ${action.title}"
  91                   description: |
  92                       ${action.description}
  93
  94                       Action triggered by clientID `${clientId}`
  95                 else:
  96                   name: "Decision Task for cron job ${cron.job_name}"
  97                   description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
  98
  99         provisionerId: "gecko-${repository.level}"
 100         workerType: "decision"
 101
 102         tags:
 103           $if: 'tasks_for == "hg-push"'
 104           then:
 105             createdForUser: "${ownerEmail}"
 106             kind: decision-task
 107           else:
 108             $if: 'tasks_for == "action"'
 109             then:
 110               createdForUser: '${ownerEmail}'
 111               kind: 'action-callback'
 112             else:
 113               $if: 'tasks_for == "cron"'
 114               then:
 115                 kind: cron-task
 116
 117         routes:
 118           $flattenDeep:
 119             - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
 120             - $if: 'tasks_for == "hg-push"'
 121               then:
 122                 - "index.gecko.v2.${repository.project}.latest.taskgraph.decision"
 123                 - "index.gecko.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
 124                 - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
 125                 - "notify.email.${ownerEmail}.on-failed"
 126                 - "notify.email.${ownerEmail}.on-exception"
 127                 # Send a notification email if the push comes from try
 128                 - $if: 'repository.project == "try"'
 129                   then:
 130                     "notify.email.${ownerEmail}.on-completed"
 131                 # These are the old index routes for the decision task.
 132                 # They are still here so external tools that referenced them continue to work.
 133                 - "index.gecko.v2.${repository.project}.latest.firefox.decision"
 134                 - "index.gecko.v2.${repository.project}.revision.${push.revision}.firefox.decision"
 135               else:
 136                 $if: 'tasks_for == "action"'
 137                 then:
 138                 - "index.gecko.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"
 139                 - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
 140                 else:  # cron
 141                 - "index.gecko.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
 142                 - "index.gecko.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
 143                 - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
 144                 # list each cron task on this revision, so actions can find them
 145                 - 'index.gecko.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
 146                 # BUG 1500166 Notify ciduty by email if a nightly hook fails
 147                 - $if: 'repository.project != "try"'
 148                   then:
 149                   - "notify.email.ciduty+failedcron@mozilla.com.on-failed"
 150                   - "notify.email.ciduty+exceptioncron@mozilla.com.on-exception"
 151                   - "notify.email.sheriffs+failedcron@mozilla.org.on-failed"
 152                   - "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception"
 153                 # These are the old index routes for the decision task.
 154                 - "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
 155
 156         scopes:
 157           $if: 'tasks_for == "hg-push"'
 158           then:
 159             - 'assume:repo:${repoUrl[8:]}:branch:default'
 160             - 'queue:route:notify.email.${ownerEmail}.*'
 161             - 'in-tree:hook-action:project-gecko/in-tree-action-${repository.level}-*'
 162           else:
 163             $if: 'tasks_for == "action"'
 164             then:
 165               # when all actions are hooks, we can calculate this directly rather than using a variable
 166               - '${action.repo_scope}'
 167             else:
 168               - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
 169
 170         dependencies: []
 171         requires: all-completed
 172
 173         priority:
 174           # Most times, there is plenty of worker capacity so everything runs
 175           # quickly, but sometimes a storm of action tasks lands.  Then we
 176           # want, from highest to lowest:
 177           # - cron tasks (time-sensitive) (low)
 178           # - action tasks (avoid interfering with the other two) (very-low)
 179           # - decision tasks (minimize user-visible delay) (lowest)
 180           # SCM levels all use different workerTypes, so there is no need for priority
 181           # between levels; "low" is the highest priority available at all levels, and
 182           # nothing runs at any higher priority on these workerTypes.
 183           $if: "tasks_for == 'cron'"
 184           then: low
 185           else:
 186             $if: "tasks_for == 'action'"
 187             then: very-low
 188             else: lowest  # tasks_for == 'hg-push'
 189         retries:
 190           $if: "tasks_for == 'hg-push'"
 191           then: 0
 192           else: 5
 193
 194         payload:
 195           env:
 196             # checkout-gecko uses these to check out the source; the inputs
 197             # to `mach taskgraph decision` are all on the command line.
 198             $merge:
 199               - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
 200                 GECKO_HEAD_REPOSITORY: '${repoUrl}'
 201                 GECKO_HEAD_REF: '${push.revision}'
 202                 GECKO_HEAD_REV: '${push.revision}'
 203                 HG_STORE_PATH: /builds/worker/checkouts/hg-store
 204                 TASKCLUSTER_CACHES: /builds/worker/checkouts
 205                 # mach generates pyc files when reading `mach_commands.py`
 206                 # This causes cached_task digest generation to be random for
 207                 # some tasks. Disable bytecode generation to work around that.
 208                 PYTHONDONTWRITEBYTECODE: '1'
 209                 # someday, these will be provided by the worker - Bug 1492664
 210                 TASKCLUSTER_ROOT_URL: https://taskcluster.net
 211                 TASKCLUSTER_PROXY_URL: http://taskcluster
 212               - $if: 'tasks_for == "action"'
 213                 then:
 214                   ACTION_TASK_GROUP_ID: '${action.taskGroupId}'     # taskGroupId of the target task
 215                   ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
 216                   ACTION_INPUT: {$json: {$eval: 'input'}}
 217                   ACTION_CALLBACK: '${action.cb_name}'
 218
 219           cache:
 220             gecko-level-${repository.level}-checkouts-sparse-v2: /builds/worker/checkouts
 221
 222           features:
 223             taskclusterProxy: true
 224             chainOfTrust: true
 225
 226           # Note: This task is built server side without the context or tooling that
 227           # exist in tree so we must hard code the hash
 228           image: 'taskcluster/decision:2.2.0@sha256:0e9689e94605eb8395f5b49141a48148416b0d825f6f7be04c29642d1a85ee3d'
 229
 230           maxRunTime: 1800
 231
 232           command:
 233             - /builds/worker/bin/run-task
 234             - '--gecko-checkout=/builds/worker/checkouts/gecko'
 235             - '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
 236             - '--'
 237             - bash
 238             - -cx
 239             - $let:
 240                 extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
 241               in:
 242                 $if: 'tasks_for == "action"'
 243                 then: >
 244                   cd /builds/worker/checkouts/gecko &&
 245                   ln -s /builds/worker/artifacts artifacts &&
 246                   ./mach --log-no-times taskgraph action-callback
 247                 else: >
 248                   cd /builds/worker/checkouts/gecko &&
 249                   ln -s /builds/worker/artifacts artifacts &&
 250                   ./mach --log-no-times taskgraph decision
 251                   --pushlog-id='${push.pushlog_id}'
 252                   --pushdate='${push.pushdate}'
 253                   --project='${repository.project}'
 254                   --owner='${ownerEmail}'
 255                   --level='${repository.level}'
 256                   --tasks-for='${tasks_for}'
 257                   --base-repository="$GECKO_BASE_REPOSITORY"
 258                   --head-repository="$GECKO_HEAD_REPOSITORY"
 259                   --head-ref="$GECKO_HEAD_REF"
 260                   --head-rev="$GECKO_HEAD_REV"
 261                   ${extraArgs}
 262
 263           artifacts:
 264             'public':
 265               type: 'directory'
 266               path: '/builds/worker/artifacts'
 267               expires: {$eval: expires}
 268
 269         extra:
 270           $merge:
 271             - treeherder:
 272                 $merge:
 273                   - machine:
 274                       platform: gecko-decision
 275                   - $if: 'tasks_for == "hg-push"'
 276                     then:
 277                       symbol: D
 278                     else:
 279                       $if: 'tasks_for == "action"'
 280                       then:
 281                         groupName: 'action-callback'
 282                         groupSymbol: AC
 283                         symbol: "${action.symbol}"
 284                       else:
 285                         groupSymbol: cron
 286                         symbol: "${cron.job_symbol}"
 287             - $if: 'tasks_for == "action"'
 288               then:
 289                 parent: '${action.taskGroupId}'
 290                 action:
 291                   name: '${action.name}'
 292                   context:
 293                     taskGroupId: '${action.taskGroupId}'
 294                     taskId: {$eval: 'taskId'}
 295                     input: {$eval: 'input'}
 296                     clientId: {$eval: 'clientId'}
 297             - $if: 'tasks_for == "cron"'
 298               then:
 299                 cron: {$json: {$eval: 'cron'}}
 300             - tasks_for: '${tasks_for}'
 301             # Email for all pushes should link to treeherder
 302             - $if: 'tasks_for == "hg-push"'
 303               then:
 304                 notify:
 305                   email:
 306                     $merge:
 307                       - link:
 308                           text: "Treeherder Jobs"
 309                           href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"
 310                       # Email for try pushes should thank you for your revision
 311                       - $if: 'repository.project == "try"'
 312                         then:
 313                           subject: "Thank you for your try submission of ${push.revision}. It's the best!"
 314                           content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs."
 315