contrib/devtools/github-merge.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2016-2017 Bitcoin Core Developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 # This script will locally construct a merge commit for a pull request on a
   7 # github repository, inspect it, sign it and optionally push it.
   8
   9 # The following temporary branches are created/overwritten and deleted:
  10 # * pull/$PULL/base (the current master we're merging onto)
  11 # * pull/$PULL/head (the current state of the remote pull request)
  12 # * pull/$PULL/merge (github's merge)
  13 # * pull/$PULL/local-merge (our merge)
  14
  15 # In case of a clean merge that is accepted by the user, the local branch with
  16 # name $BRANCH is overwritten with the merged result, and optionally pushed.
  17 from __future__ import division,print_function,unicode_literals
  18 import os
  19 from sys import stdin,stdout,stderr
  20 import argparse
  21 import hashlib
  22 import subprocess
  23 import sys
  24 import json,codecs
  25 try:
  26     from urllib.request import Request,urlopen
  27 except:
  28     from urllib2 import Request,urlopen
  29
  30 # External tools (can be overridden using environment)
  31 GIT = os.getenv('GIT','git')
  32 BASH = os.getenv('BASH','bash')
  33
  34 # OS specific configuration for terminal attributes
  35 ATTR_RESET = ''
  36 ATTR_PR = ''
  37 COMMIT_FORMAT = '%h %s (%an)%d'
  38 if os.name == 'posix': # if posix, assume we can use basic terminal escapes
  39     ATTR_RESET = '\033[0m'
  40     ATTR_PR = '\033[1;36m'
  41     COMMIT_FORMAT = '%C(bold blue)%h%Creset %s %C(cyan)(%an)%Creset%C(green)%d%Creset'
  42
  43 def git_config_get(option, default=None):
  44     '''
  45     Get named configuration option from git repository.
  46     '''
  47     try:
  48         return subprocess.check_output([GIT,'config','--get',option]).rstrip().decode('utf-8')
  49     except subprocess.CalledProcessError as e:
  50         return default
  51
  52 def retrieve_pr_info(repo,pull):
  53     '''
  54     Retrieve pull request information from github.
  55     Return None if no title can be found, or an error happens.
  56     '''
  57     try:
  58         req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull)
  59         result = urlopen(req)
  60         reader = codecs.getreader('utf-8')
  61         obj = json.load(reader(result))
  62         return obj
  63     except Exception as e:
  64         print('Warning: unable to retrieve pull information from github: %s' % e)
  65         return None
  66
  67 def ask_prompt(text):
  68     print(text,end=" ",file=stderr)
  69     stderr.flush()
  70     reply = stdin.readline().rstrip()
  71     print("",file=stderr)
  72     return reply
  73
  74 def get_symlink_files():
  75     files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', 'HEAD']).splitlines())
  76     ret = []
  77     for f in files:
  78         if (int(f.decode('utf-8').split(" ")[0], 8) & 0o170000) == 0o120000:
  79             ret.append(f.decode('utf-8').split("\t")[1])
  80     return ret
  81
  82 def tree_sha512sum(commit='HEAD'):
  83     # request metadata for entire tree, recursively
  84     files = []
  85     blob_by_name = {}
  86     for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines():
  87         name_sep = line.index(b'\t')
  88         metadata = line[:name_sep].split() # perms, 'blob', blobid
  89         assert(metadata[1] == b'blob')
  90         name = line[name_sep+1:]
  91         files.append(name)
  92         blob_by_name[name] = metadata[2]
  93
  94     files.sort()
  95     # open connection to git-cat-file in batch mode to request data for all blobs
  96     # this is much faster than launching it per file
  97     p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
  98     overall = hashlib.sha512()
  99     for f in files:
 100         blob = blob_by_name[f]
 101         # request blob
 102         p.stdin.write(blob + b'\n')
 103         p.stdin.flush()
 104         # read header: blob, "blob", size
 105         reply = p.stdout.readline().split()
 106         assert(reply[0] == blob and reply[1] == b'blob')
 107         size = int(reply[2])
 108         # hash the blob data
 109         intern = hashlib.sha512()
 110         ptr = 0
 111         while ptr < size:
 112             bs = min(65536, size - ptr)
 113             piece = p.stdout.read(bs)
 114             if len(piece) == bs:
 115                 intern.update(piece)
 116             else:
 117                 raise IOError('Premature EOF reading git cat-file output')
 118             ptr += bs
 119         dig = intern.hexdigest()
 120         assert(p.stdout.read(1) == b'\n') # ignore LF that follows blob data
 121         # update overall hash with file hash
 122         overall.update(dig.encode("utf-8"))
 123         overall.update("  ".encode("utf-8"))
 124         overall.update(f)
 125         overall.update("\n".encode("utf-8"))
 126     p.stdin.close()
 127     if p.wait():
 128         raise IOError('Non-zero return value executing git cat-file')
 129     return overall.hexdigest()
 130
 131 def print_merge_details(pull, title, branch, base_branch, head_branch):
 132     print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET))
 133     subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch])
 134
 135 def parse_arguments():
 136     epilog = '''
 137         In addition, you can set the following git configuration variables:
 138         githubmerge.repository (mandatory),
 139         user.signingkey (mandatory),
 140         githubmerge.host (default: git@github.com),
 141         githubmerge.branch (no default),
 142         githubmerge.testcmd (default: none).
 143     '''
 144     parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests',
 145             epilog=epilog)
 146     parser.add_argument('pull', metavar='PULL', type=int, nargs=1,
 147         help='Pull request ID to merge')
 148     parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?',
 149         default=None, help='Branch to merge against (default: githubmerge.branch setting, or base branch for pull, or \'master\')')
 150     return parser.parse_args()
 151
 152 def main():
 153     # Extract settings from git repo
 154     repo = git_config_get('githubmerge.repository')
 155     host = git_config_get('githubmerge.host','git@github.com')
 156     opt_branch = git_config_get('githubmerge.branch',None)
 157     testcmd = git_config_get('githubmerge.testcmd')
 158     signingkey = git_config_get('user.signingkey')
 159     if repo is None:
 160         print("ERROR: No repository configured. Use this command to set:", file=stderr)
 161         print("git config githubmerge.repository <owner>/<repo>", file=stderr)
 162         sys.exit(1)
 163     if signingkey is None:
 164         print("ERROR: No GPG signing key set. Set one using:",file=stderr)
 165         print("git config --global user.signingkey <key>",file=stderr)
 166         sys.exit(1)
 167
 168     host_repo = host+":"+repo # shortcut for push/pull target
 169
 170     # Extract settings from command line
 171     args = parse_arguments()
 172     pull = str(args.pull[0])
 173
 174     # Receive pull information from github
 175     info = retrieve_pr_info(repo,pull)
 176     if info is None:
 177         sys.exit(1)
 178     title = info['title'].strip()
 179     body = info['body'].strip()
 180     # precedence order for destination branch argument:
 181     #   - command line argument
 182     #   - githubmerge.branch setting
 183     #   - base branch for pull (as retrieved from github)
 184     #   - 'master'
 185     branch = args.branch or opt_branch or info['base']['ref'] or 'master'
 186
 187     # Initialize source branches
 188     head_branch = 'pull/'+pull+'/head'
 189     base_branch = 'pull/'+pull+'/base'
 190     merge_branch = 'pull/'+pull+'/merge'
 191     local_merge_branch = 'pull/'+pull+'/local-merge'
 192
 193     devnull = open(os.devnull,'w')
 194     try:
 195         subprocess.check_call([GIT,'checkout','-q',branch])
 196     except subprocess.CalledProcessError as e:
 197         print("ERROR: Cannot check out branch %s." % (branch), file=stderr)
 198         sys.exit(3)
 199     try:
 200         subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*'])
 201     except subprocess.CalledProcessError as e:
 202         print("ERROR: Cannot find pull request #%s on %s." % (pull,host_repo), file=stderr)
 203         sys.exit(3)
 204     try:
 205         subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout)
 206     except subprocess.CalledProcessError as e:
 207         print("ERROR: Cannot find head of pull request #%s on %s." % (pull,host_repo), file=stderr)
 208         sys.exit(3)
 209     try:
 210         subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout)
 211     except subprocess.CalledProcessError as e:
 212         print("ERROR: Cannot find merge of pull request #%s on %s." % (pull,host_repo), file=stderr)
 213         sys.exit(3)
 214     try:
 215         subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/heads/'+branch+':refs/heads/'+base_branch])
 216     except subprocess.CalledProcessError as e:
 217         print("ERROR: Cannot find branch %s on %s." % (branch,host_repo), file=stderr)
 218         sys.exit(3)
 219     subprocess.check_call([GIT,'checkout','-q',base_branch])
 220     subprocess.call([GIT,'branch','-q','-D',local_merge_branch], stderr=devnull)
 221     subprocess.check_call([GIT,'checkout','-q','-b',local_merge_branch])
 222
 223     try:
 224         # Go up to the repository's root.
 225         toplevel = subprocess.check_output([GIT,'rev-parse','--show-toplevel']).strip()
 226         os.chdir(toplevel)
 227         # Create unsigned merge commit.
 228         if title:
 229             firstline = 'Merge #%s: %s' % (pull,title)
 230         else:
 231             firstline = 'Merge #%s' % (pull,)
 232         message = firstline + '\n\n'
 233         message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]).decode('utf-8')
 234         message += '\n\nPull request description:\n\n  ' + body.replace('\n', '\n  ') + '\n'
 235         try:
 236             subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch])
 237         except subprocess.CalledProcessError as e:
 238             print("ERROR: Cannot be merged cleanly.",file=stderr)
 239             subprocess.check_call([GIT,'merge','--abort'])
 240             sys.exit(4)
 241         logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']).decode('utf-8')
 242         if logmsg.rstrip() != firstline.rstrip():
 243             print("ERROR: Creating merge failed (already merged?).",file=stderr)
 244             sys.exit(4)
 245
 246         symlink_files = get_symlink_files()
 247         for f in symlink_files:
 248             print("ERROR: File %s was a symlink" % f)
 249         if len(symlink_files) > 0:
 250             sys.exit(4)
 251
 252         # Put tree SHA512 into the message
 253         try:
 254             first_sha512 = tree_sha512sum()
 255             message += '\n\nTree-SHA512: ' + first_sha512
 256         except subprocess.CalledProcessError as e:
 257             print("ERROR: Unable to compute tree hash")
 258             sys.exit(4)
 259         try:
 260             subprocess.check_call([GIT,'commit','--amend','-m',message.encode('utf-8')])
 261         except subprocess.CalledProcessError as e:
 262             print("ERROR: Cannot update message.", file=stderr)
 263             sys.exit(4)
 264
 265         print_merge_details(pull, title, branch, base_branch, head_branch)
 266         print()
 267
 268         # Run test command if configured.
 269         if testcmd:
 270             if subprocess.call(testcmd,shell=True):
 271                 print("ERROR: Running %s failed." % testcmd,file=stderr)
 272                 sys.exit(5)
 273
 274             # Show the created merge.
 275             diff = subprocess.check_output([GIT,'diff',merge_branch+'..'+local_merge_branch])
 276             subprocess.check_call([GIT,'diff',base_branch+'..'+local_merge_branch])
 277             if diff:
 278                 print("WARNING: merge differs from github!",file=stderr)
 279                 reply = ask_prompt("Type 'ignore' to continue.")
 280                 if reply.lower() == 'ignore':
 281                     print("Difference with github ignored.",file=stderr)
 282                 else:
 283                     sys.exit(6)
 284         else:
 285             # Verify the result manually.
 286             print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr)
 287             print("Run 'git diff HEAD~' to show the changes being merged.",file=stderr)
 288             print("Type 'exit' when done.",file=stderr)
 289             if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt
 290                 os.putenv('debian_chroot',pull)
 291             subprocess.call([BASH,'-i'])
 292
 293         second_sha512 = tree_sha512sum()
 294         if first_sha512 != second_sha512:
 295             print("ERROR: Tree hash changed unexpectedly",file=stderr)
 296             sys.exit(8)
 297
 298         # Sign the merge commit.
 299         print_merge_details(pull, title, branch, base_branch, head_branch)
 300         while True:
 301             reply = ask_prompt("Type 's' to sign off on the above merge, or 'x' to reject and exit.").lower()
 302             if reply == 's':
 303                 try:
 304                     subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit'])
 305                     break
 306                 except subprocess.CalledProcessError as e:
 307                     print("Error while signing, asking again.",file=stderr)
 308             elif reply == 'x':
 309                 print("Not signing off on merge, exiting.",file=stderr)
 310                 sys.exit(1)
 311
 312         # Put the result in branch.
 313         subprocess.check_call([GIT,'checkout','-q',branch])
 314         subprocess.check_call([GIT,'reset','-q','--hard',local_merge_branch])
 315     finally:
 316         # Clean up temporary branches.
 317         subprocess.call([GIT,'checkout','-q',branch])
 318         subprocess.call([GIT,'branch','-q','-D',head_branch],stderr=devnull)
 319         subprocess.call([GIT,'branch','-q','-D',base_branch],stderr=devnull)
 320         subprocess.call([GIT,'branch','-q','-D',merge_branch],stderr=devnull)
 321         subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull)
 322
 323     # Push the result.
 324     while True:
 325         reply = ask_prompt("Type 'push' to push the result to %s, branch %s, or 'x' to exit without pushing." % (host_repo,branch)).lower()
 326         if reply == 'push':
 327             subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch])
 328             break
 329         elif reply == 'x':
 330             sys.exit(1)
 331
 332 if __name__ == '__main__':
 333     main()
 334