tools/vcs/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, unicode_literals
   6
   7 import os
   8 import re
   9 import subprocess
  10 import sys
  11
  12 import logging
  13
  14 from mach.decorators import (
  15     CommandArgument,
  16     CommandProvider,
  17     Command,
  18 )
  19
  20 from mozbuild.base import MachCommandBase
  21
  22 import mozpack.path as mozpath
  23
  24 import json
  25
  26 GITHUB_ROOT = 'https://github.com/'
  27 PR_REPOSITORIES = {
  28     'webrender': {
  29         'github': 'servo/webrender',
  30         'path': 'gfx/wr',
  31         'bugzilla_product': 'Core',
  32         'bugzilla_component': 'Graphics: WebRender',
  33     },
  34     'webgpu': {
  35         'github': 'gfx-rs/wgpu',
  36         'path': 'gfx/wgpu',
  37         'bugzilla_product': 'Core',
  38         'bugzilla_component': 'Graphics: WebGPU',
  39     },
  40     'debugger': {
  41         'github': 'firefox-devtools/debugger',
  42         'path': 'devtools/client/debugger',
  43         'bugzilla_product': 'DevTools',
  44         'bugzilla_component': 'Debugger'
  45     },
  46 }
  47
  48
  49 @CommandProvider
  50 class PullRequestImporter(MachCommandBase):
  51     @Command('import-pr', category='misc',
  52              description='Import a pull request from Github to the local repo.')
  53     @CommandArgument('-b', '--bug-number',
  54                      help='Bug number to use in the commit messages.')
  55     @CommandArgument('-t', '--bugzilla-token',
  56                      help='Bugzilla API token used to file a new bug if no bug number is '
  57                           'provided.')
  58     @CommandArgument('-r', '--reviewer',
  59                      help='Reviewer nick to apply to commit messages.')
  60     @CommandArgument('pull_request',
  61                      help='URL to the pull request to import (e.g. '
  62                           'https://github.com/servo/webrender/pull/3665).')
  63     def import_pr(self, pull_request, bug_number=None, bugzilla_token=None, reviewer=None):
  64         import requests
  65         pr_number = None
  66         repository = None
  67         for r in PR_REPOSITORIES.values():
  68             if pull_request.startswith(GITHUB_ROOT + r['github'] + '/pull/'):
  69                 # sanitize URL, dropping anything after the PR number
  70                 pr_number = int(re.search('/pull/([0-9]+)', pull_request).group(1))
  71                 pull_request = GITHUB_ROOT + r['github'] + '/pull/' + str(pr_number)
  72                 repository = r
  73                 break
  74
  75         if repository is None:
  76             self.log(logging.ERROR, 'unrecognized_repo', {},
  77                      'The pull request URL was not recognized; add it to the list of '
  78                      'recognized repos in PR_REPOSITORIES in %s' % __file__)
  79             sys.exit(1)
  80
  81         self.log(logging.INFO, 'import_pr', {'pr_url': pull_request},
  82                  'Attempting to import {pr_url}')
  83         dirty = [f for f in self.repository.get_changed_files(mode='all')
  84                  if f.startswith(repository['path'])]
  85         if dirty:
  86             self.log(logging.ERROR, 'dirty_tree', repository,
  87                      'Local {path} tree is dirty; aborting!')
  88             sys.exit(1)
  89         target_dir = mozpath.join(self.topsrcdir, os.path.normpath(repository['path']))
  90
  91         if bug_number is None:
  92             if bugzilla_token is None:
  93                 self.log(logging.WARNING, 'no_token', {},
  94                          'No bug number or bugzilla API token provided; bug number will not '
  95                          'be added to commit messages.')
  96             else:
  97                 bug_number = self._file_bug(bugzilla_token, repository, pr_number)
  98         elif bugzilla_token is not None:
  99             self.log(logging.WARNING, 'too_much_bug', {},
 100                      'Providing a bugzilla token is unnecessary when a bug number is provided. '
 101                      'Using bug number; ignoring token.')
 102
 103         pr_patch = requests.get(pull_request + '.patch')
 104         pr_patch.raise_for_status()
 105         for patch in self._split_patches(pr_patch.content, bug_number, pull_request, reviewer):
 106             self.log(logging.INFO, 'commit_msg', patch,
 107                      'Processing commit [{commit_summary}] by [{author}] at [{date}]')
 108             patch_cmd = subprocess.Popen(['patch', '-p1', '-s'], stdin=subprocess.PIPE,
 109                                          cwd=target_dir)
 110             patch_cmd.stdin.write(patch['diff'].encode('utf-8'))
 111             patch_cmd.stdin.close()
 112             patch_cmd.wait()
 113             if patch_cmd.returncode != 0:
 114                 self.log(logging.ERROR, 'commit_fail', {},
 115                          'Error applying diff from commit via "patch -p1 -s". Aborting...')
 116                 sys.exit(patch_cmd.returncode)
 117             self.repository.commit(patch['commit_msg'], patch['author'], patch['date'],
 118                                    [target_dir])
 119             self.log(logging.INFO, 'commit_pass', {},
 120                      'Committed successfully.')
 121
 122     def _file_bug(self, token, repo, pr_number):
 123         import requests
 124         bug = requests.post('https://bugzilla.mozilla.org/rest/bug?api_key=%s' % token,
 125                             json={
 126                                 'product': repo['bugzilla_product'],
 127                                 'component': repo['bugzilla_component'],
 128                                 'summary': 'Land %s#%s in mozilla-central' %
 129                                            (repo['github'], pr_number),
 130                                 'version': 'unspecified',
 131                             })
 132         bug.raise_for_status()
 133         self.log(logging.DEBUG, 'new_bug', {}, bug.content)
 134         bugnumber = json.loads(bug.content)['id']
 135         self.log(logging.INFO, 'new_bug', {'bugnumber': bugnumber},
 136                  'Filed bug {bugnumber}')
 137         return bugnumber
 138
 139     def _split_patches(self, patchfile, bug_number, pull_request, reviewer):
 140         INITIAL = 0
 141         HEADERS = 1
 142         STAT_AND_DIFF = 2
 143
 144         patch = b''
 145         state = INITIAL
 146         for line in patchfile.splitlines():
 147             if state == INITIAL:
 148                 if line.startswith(b'From '):
 149                     state = HEADERS
 150             elif state == HEADERS:
 151                 patch += line + b'\n'
 152                 if line == b'---':
 153                     state = STAT_AND_DIFF
 154             elif state == STAT_AND_DIFF:
 155                 if line.startswith(b'From '):
 156                     yield self._parse_patch(patch, bug_number, pull_request, reviewer)
 157                     patch = b''
 158                     state = HEADERS
 159                 else:
 160                     patch += line + b'\n'
 161         if len(patch) > 0:
 162             yield self._parse_patch(patch, bug_number, pull_request, reviewer)
 163         return
 164
 165     def _parse_patch(self, patch, bug_number, pull_request, reviewer):
 166         import email
 167         from email import (
 168             header,
 169             policy,
 170         )
 171
 172         parse_policy = policy.compat32.clone(max_line_length=None)
 173         parsed_mail = email.message_from_bytes(patch, policy=parse_policy)
 174
 175         def header_as_unicode(key):
 176             decoded = header.decode_header(parsed_mail[key])
 177             return str(header.make_header(decoded))
 178
 179         author = header_as_unicode('From')
 180         date = header_as_unicode('Date')
 181         commit_summary = header_as_unicode('Subject')
 182         email_body = parsed_mail.get_payload(decode=True).decode('utf-8')
 183         (commit_body, diff) = ('\n' + email_body).rsplit('\n---\n', 1)
 184
 185         bug_prefix = ''
 186         if bug_number is not None:
 187             bug_prefix = 'Bug %s - ' % bug_number
 188         commit_summary = re.sub(r'^\[PATCH[0-9 /]*\] ', bug_prefix, commit_summary)
 189         if reviewer is not None:
 190             commit_summary += ' r=' + reviewer
 191
 192         commit_msg = commit_summary + '\n'
 193         if len(commit_body) > 0:
 194             commit_msg += commit_body + '\n'
 195         commit_msg += '\n[import_pr] From ' + pull_request + '\n'
 196
 197         patch_obj = {
 198             'author': author,
 199             'date': date,
 200             'commit_summary': commit_summary,
 201             'commit_msg': commit_msg,
 202             'diff': diff,
 203         }
 204         return patch_obj