.taskcluster.yml

   1 # yamllint disable rule:line-length
   2 # This file is rendered via JSON-e by
   3 # - hg-push - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/hg_push.py
   4 #   {
   5 #     tasks_for: 'hg-push',
   6 #     push: {owner, comment, pushlog_id, pushdate},
   7 #     repository: {url, project, level},
   8 #     now,
   9 #     as_slugid: // function
  10 #     ownTaskId: // taskId of the task that will be created
  11 #   }
  12 #
  13 # - cron tasks - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/cron/decision.py
  14 #   {
  15 #     tasks_for: 'cron',
  16 #     push: {revision, pushlog_id, pushdate, owner}
  17 #     repository: {url, project, level},
  18 #     cron: {task_id, job_name, job_symbol, quoted_args},
  19 #     now,
  20 #     ownTaskId: // taskId of the task that will be created
  21 #   }
  22 #
  23 # - action tasks - See:
  24 #   * taskcluster/gecko_taskgraph/actions/registry.py,
  25 #   * https://docs.taskcluster.net/docs/manual/using/actions/spec
  26 #   * ci-admin:ciadmin/generate/in_tree_actions.py
  27 #
  28 #   The registry generates the hookPayload that appears in actions.json, and
  29 #   contains data from the decision task as well as JSON-e code to combine that
  30 #   with data supplied as part of the action spec.  When the hook is fired, the
  31 #   hookPayload is rendered with JSON-e to produce a payload for the hook task
  32 #   template.
  33 #
  34 #   The ci-admin code wraps the content of this file (.taskcluster.yml) with a
  35 #   JSON-e $let statement that produces the context described below, and
  36 #   installs that as the hook task template.
  37 #
  38 #   {
  39 #     tasks_for: 'action',
  40 #     push: {owner, pushlog_id, revision, base_revision},
  41 #     repository: {url, project, level},
  42 #     input,
  43 #     taskId,      // targetted taskId
  44 #     taskGroupId, // targetted taskGroupId
  45 #     action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
  46 #     ownTaskId:   // taskId of the task that will be created
  47 #     clientId:    // clientId that triggered this hook
  48 #   }
  49 ---
  50 version: 1
  51 tasks:
  52     # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
  53     # takes place.
  54     - $if: 'tasks_for in ["hg-push", "action", "cron"]'
  55       then:
  56           $let:
  57               # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
  58               ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
  59               # ensure there's no trailing `/` on the repo URL
  60               repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
  61               # expire try earlier than other branches
  62               expires:
  63                   $if: 'repository.project == "try"'
  64                   then: {$fromNow: '28 days'}
  65                   else: {$fromNow: '1 year'}
  66               trustDomain: gecko
  67               treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})'
  68           in:
  69               taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
  70               taskGroupId:
  71                   $if: 'tasks_for == "action"'
  72                   then:
  73                       '${action.taskGroupId}'
  74                   else:
  75                       '${ownTaskId}'  # same as taskId; this is how automation identifies a decision task
  76               schedulerId: '${trustDomain}-level-${repository.level}'
  77
  78               created: {$fromNow: ''}
  79               deadline: {$fromNow: '1 day'}
  80               expires: {$eval: 'expires'}
  81               metadata:
  82                   $merge:
  83                       - owner: "${ownerEmail}"
  84                         source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
  85                       - $if: 'tasks_for == "hg-push"'
  86                         then:
  87                             name: "Gecko Decision Task"
  88                             description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})'
  89                         else:
  90                             $if: 'tasks_for == "action"'
  91                             then:
  92                                 name: "Action: ${action.title}"
  93                                 description: |
  94                                     ${action.description}
  95
  96                                     ${treeherder_link}
  97
  98                                     Action triggered by clientID `${clientId}`
  99                             else:
 100                                 name: "Decision Task for cron job ${cron.job_name}"
 101                                 description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})'
 102
 103               provisionerId: "${trustDomain}-${repository.level}"
 104               workerType: "decision-gcp"
 105
 106               tags:
 107                   $if: 'tasks_for == "hg-push"'
 108                   then:
 109                       createdForUser: "${ownerEmail}"
 110                       kind: decision-task
 111                   else:
 112                       $if: 'tasks_for == "action"'
 113                       then:
 114                           createdForUser: '${ownerEmail}'
 115                           kind: 'action-callback'
 116                       else:
 117                           $if: 'tasks_for == "cron"'
 118                           then:
 119                               kind: cron-task
 120
 121               routes:
 122                   $flattenDeep:
 123                       - "tc-treeherder.v2.${repository.project}.${push.revision}"
 124                       - $if: 'tasks_for == "hg-push"'
 125                         then:
 126                             - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"
 127                             - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
 128                             - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
 129                             - "notify.email.${ownerEmail}.on-failed"
 130                             - "notify.email.${ownerEmail}.on-exception"
 131                             # Send a notification email if the push comes from try
 132                             - $if: 'repository.project == "try"'
 133                               then:
 134                                   - "notify.email.${ownerEmail}.on-completed"
 135                             - $if: 'repository.project == "mozilla-central"'
 136                               then:
 137                                   # Notify #thunderbird-ci
 138                                   - "notify.matrix-room.!TWztIhgqLawNpRBZTC:mozilla.org.on-completed"
 139                         else:
 140                             $if: 'tasks_for == "action"'
 141                             then:
 142                                 - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"
 143                                 - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
 144                             else:  # cron
 145                                 - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
 146                                 - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
 147                                 - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
 148                                 # list each cron task on this revision, so actions can find them
 149                                 - 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
 150                                 # BUG 1500166 Notify ciduty by email if a nightly hook fails
 151                                 - $if: 'repository.project != "try"'
 152                                   then:
 153                                       - "notify.email.ciduty+failedcron@mozilla.com.on-failed"
 154                                       - "notify.email.ciduty+exceptioncron@mozilla.com.on-exception"
 155                                       - "notify.email.sheriffs+failedcron@mozilla.org.on-failed"
 156                                       - "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception"
 157
 158               scopes:
 159                   $if: 'tasks_for == "hg-push"'
 160                   then:
 161                       - 'assume:repo:${repoUrl[8:]}:branch:default'
 162                       - 'queue:route:notify.email.${ownerEmail}.*'
 163                       - 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*'
 164                       - 'index:insert-task:${trustDomain}.v2.${repository.project}.*'
 165                   else:
 166                       $if: 'tasks_for == "action"'
 167                       then:
 168                           # when all actions are hooks, we can calculate this directly rather than using a variable
 169                           - '${action.repo_scope}'
 170                       else:
 171                           - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
 172
 173               dependencies: []
 174               requires: all-completed
 175
 176               priority:
 177                   # Most times, there is plenty of worker capacity so everything runs
 178                   # quickly, but sometimes a storm of action tasks lands.  Then we
 179                   # want, from highest to lowest:
 180                   # - cron tasks (time-sensitive) (low)
 181                   # - action tasks (avoid interfering with the other two) (very-low)
 182                   # - decision tasks (minimize user-visible delay) (lowest)
 183                   # SCM levels all use different workerTypes, so there is no need for priority
 184                   # between levels; "low" is the highest priority available at all levels, and
 185                   # nothing runs at any higher priority on these workerTypes.
 186                   $if: "tasks_for == 'cron'"
 187                   then: low
 188                   else:
 189                       $if: "tasks_for == 'action'"
 190                       then: very-low
 191                       else: lowest  # tasks_for == 'hg-push'
 192               retries:
 193                   $if: "tasks_for == 'hg-push'"
 194                   then: 0
 195                   else: 5
 196
 197               payload:
 198                   env:
 199                       # run-task uses these to check out the source; the inputs
 200                       # to `mach taskgraph decision` are all on the command line.
 201                       $merge:
 202                           - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
 203                             GECKO_BASE_REV: '${push.base_revision}'
 204                             GECKO_HEAD_REPOSITORY: '${repoUrl}'
 205                             GECKO_HEAD_REF: '${push.revision}'
 206                             GECKO_HEAD_REV: '${push.revision}'
 207                             HG_STORE_PATH: /builds/worker/checkouts/hg-store
 208                             TASKCLUSTER_CACHES: /builds/worker/checkouts
 209                             MOZ_AUTOMATION: '1'
 210                             # mach generates pyc files when reading `mach_commands.py`
 211                             # This causes cached_task digest generation to be random for
 212                             # some tasks. Disable bytecode generation to work around that.
 213                             PYTHONDONTWRITEBYTECODE: '1'
 214                           - $if: 'tasks_for == "action"'
 215                             then:
 216                                 ACTION_TASK_GROUP_ID: '${action.taskGroupId}'  # taskGroupId of the target task
 217                                 ACTION_TASK_ID: {$json: {$eval: 'taskId'}}  # taskId of the target task (JSON-encoded)
 218                                 ACTION_INPUT: {$json: {$eval: 'input'}}
 219                                 ACTION_CALLBACK: '${action.cb_name}'
 220
 221                   cache:
 222                       "${trustDomain}-level-${repository.level}-checkouts-sparse-v4": /builds/worker/checkouts
 223
 224                   features:
 225                       taskclusterProxy: true
 226                       chainOfTrust: true
 227
 228                   # Note: This task is built server side without the context or tooling that
 229                   # exist in tree so we must hard code the hash
 230                   image: 'mozillareleases/gecko_decision:4.1.0@sha256:ea73900129328095da4fb033697690d538a08d935298fef4d5fa4aae80273bce'
 231
 232                   maxRunTime: 3600
 233
 234                   command:
 235                       - /builds/worker/bin/run-task
 236                       - '--gecko-checkout=/builds/worker/checkouts/gecko'
 237                       - '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
 238                       - '--'
 239                       - bash
 240                       - -cx
 241                       - $let:
 242                             extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
 243                         in:
 244                             $if: 'tasks_for == "action"'
 245                             then: >
 246                                 cd /builds/worker/checkouts/gecko &&
 247                                 ln -s /builds/worker/artifacts artifacts &&
 248                                 ./mach --log-no-times taskgraph action-callback
 249                             else: >
 250                                 cd /builds/worker/checkouts/gecko &&
 251                                 ln -s /builds/worker/artifacts artifacts &&
 252                                 ./mach --log-no-times taskgraph decision
 253                                 --pushlog-id='${push.pushlog_id}'
 254                                 --pushdate='${push.pushdate}'
 255                                 --project='${repository.project}'
 256                                 --owner='${ownerEmail}'
 257                                 --level='${repository.level}'
 258                                 --tasks-for='${tasks_for}'
 259                                 --repository-type=hg
 260                                 --base-repository="$GECKO_BASE_REPOSITORY"
 261                                 --base-rev="$GECKO_BASE_REV"
 262                                 --head-repository="$GECKO_HEAD_REPOSITORY"
 263                                 --head-ref="$GECKO_HEAD_REF"
 264                                 --head-rev="$GECKO_HEAD_REV"
 265                                 ${extraArgs}
 266
 267                   artifacts:
 268                       'public':
 269                           type: 'directory'
 270                           path: '/builds/worker/artifacts'
 271                           expires: {$eval: expires}
 272                       'public/docker-contexts':
 273                           type: 'directory'
 274                           path: '/builds/worker/checkouts/gecko/docker-contexts'
 275                           # This needs to be at least the deadline of the
 276                           # decision task + the docker-image task deadlines.
 277                           # It is set to a week to allow for some time for
 278                           # debugging, but they are not useful long-term.
 279                           expires: {$fromNow: '7 day'}
 280
 281               extra:
 282                   $merge:
 283                       - treeherder:
 284                             $merge:
 285                                 - machine:
 286                                       platform: gecko-decision
 287                                 - $if: 'tasks_for == "hg-push"'
 288                                   then:
 289                                       symbol: D
 290                                   else:
 291                                       $if: 'tasks_for == "action"'
 292                                       then:
 293                                           groupName: 'action-callback'
 294                                           groupSymbol: AC
 295                                           symbol: "${action.symbol}"
 296                                       else:
 297                                           groupSymbol: cron
 298                                           symbol: "${cron.job_symbol}"
 299                       - $if: 'tasks_for == "action"'
 300                         then:
 301                             parent: '${action.taskGroupId}'
 302                             action:
 303                                 name: '${action.name}'
 304                                 context:
 305                                     taskGroupId: '${action.taskGroupId}'
 306                                     taskId: {$eval: 'taskId'}
 307                                     input: {$eval: 'input'}
 308                                     clientId: {$eval: 'clientId'}
 309                       - $if: 'tasks_for == "cron"'
 310                         then:
 311                             cron: {$json: {$eval: 'cron'}}
 312                       - tasks_for: '${tasks_for}'
 313                       # Email for all pushes should link to treeherder
 314                       - $if: 'tasks_for == "hg-push"'
 315                         then:
 316                             notify:
 317                                 $merge:
 318                                     - email:
 319                                           $merge:
 320                                               - link:
 321                                                     text: "Treeherder Jobs"
 322                                                     href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"
 323                                               - $if: 'repository.project == "try"'
 324                                                 then:
 325                                                     subject: "Thank you for your try submission of ${push.revision}. It's the best!"
 326                                                     content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs."
 327                                     - $if: 'repository.project == "mozilla-central"'
 328                                       then:
 329                                           matrixBody: "${repository.project} push notification: https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"