.taskcluster.yml

   1 # yamllint disable rule:line-length
   2 # This file is rendered via JSON-e by
   3 # - hg-push - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/hg_push.py
   4 #   {
   5 #     tasks_for: 'hg-push',
   6 #     push: {owner, comment, pushlog_id, pushdate},
   7 #     repository: {url, project, level},
   8 #     now,
   9 #     as_slugid: // function
  10 #     ownTaskId: // taskId of the task that will be created
  11 #   }
  12 #
  13 # - cron tasks - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/cron/decision.py
  14 #   {
  15 #     tasks_for: 'cron',
  16 #     push: {revision, pushlog_id, pushdate, owner}
  17 #     repository: {url, project, level},
  18 #     cron: {task_id, job_name, job_symbol, quoted_args},
  19 #     now,
  20 #     ownTaskId: // taskId of the task that will be created
  21 #   }
  22 #
  23 # - action tasks - See:
  24 #   * taskcluster/gecko_taskgraph/actions/registry.py,
  25 #   * https://docs.taskcluster.net/docs/manual/using/actions/spec
  26 #   * ci-admin:ciadmin/generate/in_tree_actions.py
  27 #
  28 #   The registry generates the hookPayload that appears in actions.json, and
  29 #   contains data from the decision task as well as JSON-e code to combine that
  30 #   with data supplied as part of the action spec.  When the hook is fired, the
  31 #   hookPayload is rendered with JSON-e to produce a payload for the hook task
  32 #   template.
  33 #
  34 #   The ci-admin code wraps the content of this file (.taskcluster.yml) with a
  35 #   JSON-e $let statement that produces the context described below, and
  36 #   installs that as the hook task template.
  37 #
  38 #   {
  39 #     tasks_for: 'action',
  40 #     push: {owner, pushlog_id, revision, base_revision},
  41 #     repository: {url, project, level},
  42 #     input,
  43 #     taskId,      // targetted taskId
  44 #     taskGroupId, // targetted taskGroupId
  45 #     action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
  46 #     ownTaskId:   // taskId of the task that will be created
  47 #     clientId:    // clientId that triggered this hook
  48 #   }
  49 ---
  50 version: 1
  51 tasks:
  52     # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
  53     # takes place.
  54     - $if: 'tasks_for in ["hg-push", "action", "cron"]'
  55       then:
  56           $let:
  57               # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
  58               ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
  59               # ensure there's no trailing `/` on the repo URL
  60               repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
  61               # expire try earlier than other branches
  62               expires:
  63                   $if: 'repository.project == "try"'
  64                   then: {$fromNow: '28 days'}
  65                   else: {$fromNow: '1 year'}
  66               trustDomain: gecko
  67               treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})'
  68           in:
  69               taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
  70               taskGroupId:
  71                   $if: 'tasks_for == "action"'
  72                   then:
  73                       '${action.taskGroupId}'
  74                   else:
  75                       '${ownTaskId}'  # same as taskId; this is how automation identifies a decision task
  76               schedulerId: '${trustDomain}-level-${repository.level}'
  77
  78               created: {$fromNow: ''}
  79               deadline: {$fromNow: '1 day'}
  80               expires: {$eval: 'expires'}
  81               metadata:
  82                   $merge:
  83                       - owner: "${ownerEmail}"
  84                         source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
  85                       - $if: 'tasks_for == "hg-push"'
  86                         then:
  87                             name: "Gecko Decision Task"
  88                             description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})'
  89                         else:
  90                             $if: 'tasks_for == "action"'
  91                             then:
  92                                 name: "Action: ${action.title}"
  93                                 description: |
  94                                     ${action.description}
  95
  96                                     ${treeherder_link}
  97
  98                                     Action triggered by clientID `${clientId}`
  99                             else:
 100                                 name: "Decision Task for cron job ${cron.job_name}"
 101                                 description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})'
 102
 103               provisionerId: "${trustDomain}-${repository.level}"
 104               workerType: "decision-gcp"
 105
 106               tags:
 107                   $if: 'tasks_for == "hg-push"'
 108                   then:
 109                       createdForUser: "${ownerEmail}"
 110                       kind: decision-task
 111                   else:
 112                       $if: 'tasks_for == "action"'
 113                       then:
 114                           createdForUser: '${ownerEmail}'
 115                           kind: 'action-callback'
 116                       else:
 117                           $if: 'tasks_for == "cron"'
 118                           then:
 119                               kind: cron-task
 120
 121               routes:
 122                   $flattenDeep:
 123                       - "tc-treeherder.v2.${repository.project}.${push.revision}"
 124                       - $if: 'tasks_for == "hg-push"'
 125                         then:
 126                             - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"
 127                             - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
 128                             - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
 129                             - "notify.email.${ownerEmail}.on-failed"
 130                             - "notify.email.${ownerEmail}.on-exception"
 131                             # Send a notification email if the push comes from try
 132                             - $if: 'repository.project == "try"'
 133                               then:
 134                                   "notify.email.${ownerEmail}.on-completed"
 135                         else:
 136                             $if: 'tasks_for == "action"'
 137                             then:
 138                                 - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"
 139                                 - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
 140                             else:  # cron
 141                                 - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
 142                                 - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
 143                                 - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
 144                                 # list each cron task on this revision, so actions can find them
 145                                 - 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
 146                                 # BUG 1500166 Notify ciduty by email if a nightly hook fails
 147                                 - $if: 'repository.project != "try"'
 148                                   then:
 149                                       - "notify.email.ciduty+failedcron@mozilla.com.on-failed"
 150                                       - "notify.email.ciduty+exceptioncron@mozilla.com.on-exception"
 151                                       - "notify.email.sheriffs+failedcron@mozilla.org.on-failed"
 152                                       - "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception"
 153
 154               scopes:
 155                   $if: 'tasks_for == "hg-push"'
 156                   then:
 157                       - 'assume:repo:${repoUrl[8:]}:branch:default'
 158                       - 'queue:route:notify.email.${ownerEmail}.*'
 159                       - 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*'
 160                       - 'index:insert-task:${trustDomain}.v2.${repository.project}.*'
 161                   else:
 162                       $if: 'tasks_for == "action"'
 163                       then:
 164                           # when all actions are hooks, we can calculate this directly rather than using a variable
 165                           - '${action.repo_scope}'
 166                       else:
 167                           - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
 168
 169               dependencies: []
 170               requires: all-completed
 171
 172               priority:
 173                   # Most times, there is plenty of worker capacity so everything runs
 174                   # quickly, but sometimes a storm of action tasks lands.  Then we
 175                   # want, from highest to lowest:
 176                   # - cron tasks (time-sensitive) (low)
 177                   # - action tasks (avoid interfering with the other two) (very-low)
 178                   # - decision tasks (minimize user-visible delay) (lowest)
 179                   # SCM levels all use different workerTypes, so there is no need for priority
 180                   # between levels; "low" is the highest priority available at all levels, and
 181                   # nothing runs at any higher priority on these workerTypes.
 182                   $if: "tasks_for == 'cron'"
 183                   then: low
 184                   else:
 185                       $if: "tasks_for == 'action'"
 186                       then: very-low
 187                       else: lowest  # tasks_for == 'hg-push'
 188               retries:
 189                   $if: "tasks_for == 'hg-push'"
 190                   then: 0
 191                   else: 5
 192
 193               payload:
 194                   env:
 195                       # run-task uses these to check out the source; the inputs
 196                       # to `mach taskgraph decision` are all on the command line.
 197                       $merge:
 198                           - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
 199                             GECKO_BASE_REV: '${push.base_revision}'
 200                             GECKO_HEAD_REPOSITORY: '${repoUrl}'
 201                             GECKO_HEAD_REF: '${push.revision}'
 202                             GECKO_HEAD_REV: '${push.revision}'
 203                             HG_STORE_PATH: /builds/worker/checkouts/hg-store
 204                             TASKCLUSTER_CACHES: /builds/worker/checkouts
 205                             MOZ_AUTOMATION: '1'
 206                             # mach generates pyc files when reading `mach_commands.py`
 207                             # This causes cached_task digest generation to be random for
 208                             # some tasks. Disable bytecode generation to work around that.
 209                             PYTHONDONTWRITEBYTECODE: '1'
 210                           - $if: 'tasks_for == "action"'
 211                             then:
 212                                 ACTION_TASK_GROUP_ID: '${action.taskGroupId}'  # taskGroupId of the target task
 213                                 ACTION_TASK_ID: {$json: {$eval: 'taskId'}}  # taskId of the target task (JSON-encoded)
 214                                 ACTION_INPUT: {$json: {$eval: 'input'}}
 215                                 ACTION_CALLBACK: '${action.cb_name}'
 216
 217                   cache:
 218                       "${trustDomain}-level-${repository.level}-checkouts-sparse-v3": /builds/worker/checkouts
 219
 220                   features:
 221                       taskclusterProxy: true
 222                       chainOfTrust: true
 223
 224                   # Note: This task is built server side without the context or tooling that
 225                   # exist in tree so we must hard code the hash
 226                   image: 'mozillareleases/gecko_decision:4.0.0@sha256:9f69fe08c28e3cb3cc296451f0a2735df6e25d0e3c877ea735ef1b7f0b345b06'
 227
 228                   maxRunTime: 3600
 229
 230                   command:
 231                       - /builds/worker/bin/run-task
 232                       - '--gecko-checkout=/builds/worker/checkouts/gecko'
 233                       - '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
 234                       - '--'
 235                       - bash
 236                       - -cx
 237                       - $let:
 238                             extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
 239                         in:
 240                             $if: 'tasks_for == "action"'
 241                             then: >
 242                                 cd /builds/worker/checkouts/gecko &&
 243                                 ln -s /builds/worker/artifacts artifacts &&
 244                                 ./mach --log-no-times taskgraph action-callback
 245                             else: >
 246                                 cd /builds/worker/checkouts/gecko &&
 247                                 ln -s /builds/worker/artifacts artifacts &&
 248                                 ./mach --log-no-times taskgraph decision
 249                                 --pushlog-id='${push.pushlog_id}'
 250                                 --pushdate='${push.pushdate}'
 251                                 --project='${repository.project}'
 252                                 --owner='${ownerEmail}'
 253                                 --level='${repository.level}'
 254                                 --tasks-for='${tasks_for}'
 255                                 --repository-type=hg
 256                                 --base-repository="$GECKO_BASE_REPOSITORY"
 257                                 --base-rev="$GECKO_BASE_REV"
 258                                 --head-repository="$GECKO_HEAD_REPOSITORY"
 259                                 --head-ref="$GECKO_HEAD_REF"
 260                                 --head-rev="$GECKO_HEAD_REV"
 261                                 ${extraArgs}
 262
 263                   artifacts:
 264                       'public':
 265                           type: 'directory'
 266                           path: '/builds/worker/artifacts'
 267                           expires: {$eval: expires}
 268                       'public/docker-contexts':
 269                           type: 'directory'
 270                           path: '/builds/worker/checkouts/gecko/docker-contexts'
 271                           # This needs to be at least the deadline of the
 272                           # decision task + the docker-image task deadlines.
 273                           # It is set to a week to allow for some time for
 274                           # debugging, but they are not useful long-term.
 275                           expires: {$fromNow: '7 day'}
 276
 277               extra:
 278                   $merge:
 279                       - treeherder:
 280                             $merge:
 281                                 - machine:
 282                                       platform: gecko-decision
 283                                 - $if: 'tasks_for == "hg-push"'
 284                                   then:
 285                                       symbol: D
 286                                   else:
 287                                       $if: 'tasks_for == "action"'
 288                                       then:
 289                                           groupName: 'action-callback'
 290                                           groupSymbol: AC
 291                                           symbol: "${action.symbol}"
 292                                       else:
 293                                           groupSymbol: cron
 294                                           symbol: "${cron.job_symbol}"
 295                       - $if: 'tasks_for == "action"'
 296                         then:
 297                             parent: '${action.taskGroupId}'
 298                             action:
 299                                 name: '${action.name}'
 300                                 context:
 301                                     taskGroupId: '${action.taskGroupId}'
 302                                     taskId: {$eval: 'taskId'}
 303                                     input: {$eval: 'input'}
 304                                     clientId: {$eval: 'clientId'}
 305                       - $if: 'tasks_for == "cron"'
 306                         then:
 307                             cron: {$json: {$eval: 'cron'}}
 308                       - tasks_for: '${tasks_for}'
 309                       # Email for all pushes should link to treeherder
 310                       - $if: 'tasks_for == "hg-push"'
 311                         then:
 312                             notify:
 313                                 email:
 314                                     $merge:
 315                                         - link:
 316                                               text: "Treeherder Jobs"
 317                                               href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"
 318                                         # Email for try pushes should thank you for your revision
 319                                         - $if: 'repository.project == "try"'
 320                                           then:
 321                                               subject: "Thank you for your try submission of ${push.revision}. It's the best!"
 322                                               content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs."