tools/vcs/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, unicode_literals
   6
   7 import os
   8 import re
   9 import subprocess
  10 import sys
  11
  12 import logging
  13
  14 from mach.decorators import (
  15     CommandArgument,
  16     Command,
  17 )
  18
  19 import mozpack.path as mozpath
  20
  21 import json
  22
  23 GITHUB_ROOT = "https://github.com/"
  24 PR_REPOSITORIES = {
  25     "webrender": {
  26         "github": "servo/webrender",
  27         "path": "gfx/wr",
  28         "bugzilla_product": "Core",
  29         "bugzilla_component": "Graphics: WebRender",
  30     },
  31     "webgpu": {
  32         "github": "gfx-rs/wgpu",
  33         "path": "gfx/wgpu",
  34         "bugzilla_product": "Core",
  35         "bugzilla_component": "Graphics: WebGPU",
  36     },
  37     "debugger": {
  38         "github": "firefox-devtools/debugger",
  39         "path": "devtools/client/debugger",
  40         "bugzilla_product": "DevTools",
  41         "bugzilla_component": "Debugger",
  42     },
  43 }
  44
  45
  46 @Command(
  47     "import-pr",
  48     category="misc",
  49     description="Import a pull request from Github to the local repo.",
  50 )
  51 @CommandArgument("-b", "--bug-number", help="Bug number to use in the commit messages.")
  52 @CommandArgument(
  53     "-t",
  54     "--bugzilla-token",
  55     help="Bugzilla API token used to file a new bug if no bug number is provided.",
  56 )
  57 @CommandArgument("-r", "--reviewer", help="Reviewer nick to apply to commit messages.")
  58 @CommandArgument(
  59     "pull_request",
  60     help="URL to the pull request to import (e.g. "
  61     "https://github.com/servo/webrender/pull/3665).",
  62 )
  63 def import_pr(
  64     command_context,
  65     pull_request,
  66     bug_number=None,
  67     bugzilla_token=None,
  68     reviewer=None,
  69 ):
  70     import requests
  71
  72     pr_number = None
  73     repository = None
  74     for r in PR_REPOSITORIES.values():
  75         if pull_request.startswith(GITHUB_ROOT + r["github"] + "/pull/"):
  76             # sanitize URL, dropping anything after the PR number
  77             pr_number = int(re.search("/pull/([0-9]+)", pull_request).group(1))
  78             pull_request = GITHUB_ROOT + r["github"] + "/pull/" + str(pr_number)
  79             repository = r
  80             break
  81
  82     if repository is None:
  83         command_context.log(
  84             logging.ERROR,
  85             "unrecognized_repo",
  86             {},
  87             "The pull request URL was not recognized; add it to the list of "
  88             "recognized repos in PR_REPOSITORIES in %s" % __file__,
  89         )
  90         sys.exit(1)
  91
  92     command_context.log(
  93         logging.INFO,
  94         "import_pr",
  95         {"pr_url": pull_request},
  96         "Attempting to import {pr_url}",
  97     )
  98     dirty = [
  99         f
 100         for f in command_context.repository.get_changed_files(mode="all")
 101         if f.startswith(repository["path"])
 102     ]
 103     if dirty:
 104         command_context.log(
 105             logging.ERROR,
 106             "dirty_tree",
 107             repository,
 108             "Local {path} tree is dirty; aborting!",
 109         )
 110         sys.exit(1)
 111     target_dir = mozpath.join(
 112         command_context.topsrcdir, os.path.normpath(repository["path"])
 113     )
 114
 115     if bug_number is None:
 116         if bugzilla_token is None:
 117             command_context.log(
 118                 logging.WARNING,
 119                 "no_token",
 120                 {},
 121                 "No bug number or bugzilla API token provided; bug number will not "
 122                 "be added to commit messages.",
 123             )
 124         else:
 125             bug_number = _file_bug(
 126                 command_context, bugzilla_token, repository, pr_number
 127             )
 128     elif bugzilla_token is not None:
 129         command_context.log(
 130             logging.WARNING,
 131             "too_much_bug",
 132             {},
 133             "Providing a bugzilla token is unnecessary when a bug number is provided. "
 134             "Using bug number; ignoring token.",
 135         )
 136
 137     pr_patch = requests.get(pull_request + ".patch")
 138     pr_patch.raise_for_status()
 139     for patch in _split_patches(pr_patch.content, bug_number, pull_request, reviewer):
 140         command_context.log(
 141             logging.INFO,
 142             "commit_msg",
 143             patch,
 144             "Processing commit [{commit_summary}] by [{author}] at [{date}]",
 145         )
 146         patch_cmd = subprocess.Popen(
 147             ["patch", "-p1", "-s"], stdin=subprocess.PIPE, cwd=target_dir
 148         )
 149         patch_cmd.stdin.write(patch["diff"].encode("utf-8"))
 150         patch_cmd.stdin.close()
 151         patch_cmd.wait()
 152         if patch_cmd.returncode != 0:
 153             command_context.log(
 154                 logging.ERROR,
 155                 "commit_fail",
 156                 {},
 157                 'Error applying diff from commit via "patch -p1 -s". Aborting...',
 158             )
 159             sys.exit(patch_cmd.returncode)
 160         command_context.repository.commit(
 161             patch["commit_msg"], patch["author"], patch["date"], [target_dir]
 162         )
 163         command_context.log(logging.INFO, "commit_pass", {}, "Committed successfully.")
 164
 165
 166 def _file_bug(command_context, token, repo, pr_number):
 167     import requests
 168
 169     bug = requests.post(
 170         "https://bugzilla.mozilla.org/rest/bug?api_key=%s" % token,
 171         json={
 172             "product": repo["bugzilla_product"],
 173             "component": repo["bugzilla_component"],
 174             "summary": "Land %s#%s in mozilla-central" % (repo["github"], pr_number),
 175             "version": "unspecified",
 176         },
 177     )
 178     bug.raise_for_status()
 179     command_context.log(logging.DEBUG, "new_bug", {}, bug.content)
 180     bugnumber = json.loads(bug.content)["id"]
 181     command_context.log(
 182         logging.INFO, "new_bug", {"bugnumber": bugnumber}, "Filed bug {bugnumber}"
 183     )
 184     return bugnumber
 185
 186
 187 def _split_patches(patchfile, bug_number, pull_request, reviewer):
 188     INITIAL = 0
 189     HEADERS = 1
 190     STAT_AND_DIFF = 2
 191
 192     patch = b""
 193     state = INITIAL
 194     for line in patchfile.splitlines():
 195         if state == INITIAL:
 196             if line.startswith(b"From "):
 197                 state = HEADERS
 198         elif state == HEADERS:
 199             patch += line + b"\n"
 200             if line == b"---":
 201                 state = STAT_AND_DIFF
 202         elif state == STAT_AND_DIFF:
 203             if line.startswith(b"From "):
 204                 yield _parse_patch(patch, bug_number, pull_request, reviewer)
 205                 patch = b""
 206                 state = HEADERS
 207             else:
 208                 patch += line + b"\n"
 209     if len(patch) > 0:
 210         yield _parse_patch(patch, bug_number, pull_request, reviewer)
 211     return
 212
 213
 214 def _parse_patch(patch, bug_number, pull_request, reviewer):
 215     import email
 216     from email import (
 217         header,
 218         policy,
 219     )
 220
 221     parse_policy = policy.compat32.clone(max_line_length=None)
 222     parsed_mail = email.message_from_bytes(patch, policy=parse_policy)
 223
 224     def header_as_unicode(key):
 225         decoded = header.decode_header(parsed_mail[key])
 226         return str(header.make_header(decoded))
 227
 228     author = header_as_unicode("From")
 229     date = header_as_unicode("Date")
 230     commit_summary = header_as_unicode("Subject")
 231     email_body = parsed_mail.get_payload(decode=True).decode("utf-8")
 232     (commit_body, diff) = ("\n" + email_body).rsplit("\n---\n", 1)
 233
 234     bug_prefix = ""
 235     if bug_number is not None:
 236         bug_prefix = "Bug %s - " % bug_number
 237     commit_summary = re.sub(r"^\[PATCH[0-9 /]*\] ", bug_prefix, commit_summary)
 238     if reviewer is not None:
 239         commit_summary += " r=" + reviewer
 240
 241     commit_msg = commit_summary + "\n"
 242     if len(commit_body) > 0:
 243         commit_msg += commit_body + "\n"
 244     commit_msg += "\n[import_pr] From " + pull_request + "\n"
 245
 246     patch_obj = {
 247         "author": author,
 248         "date": date,
 249         "commit_summary": commit_summary,
 250         "commit_msg": commit_msg,
 251         "diff": diff,
 252     }
 253     return patch_obj