cola/gitcmds.py

   1 """Provides commands and queries for Git."""
   2 from __future__ import division, absolute_import, unicode_literals
   3
   4 import re
   5 from io import StringIO
   6
   7 from . import core
   8 from . import gitcfg
   9 from . import utils
  10 from . import version
  11 from .git import git
  12 from .git import STDOUT
  13 from .i18n import N_
  14
  15
  16 EMPTY_TREE_OID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'
  17
  18
  19 class InvalidRepositoryError(Exception):
  20     pass
  21
  22
  23 def default_remote(config=None):
  24     """Return the remote tracked by the current branch."""
  25     if config is None:
  26         config = gitcfg.current()
  27     return config.get('branch.%s.remote' % current_branch())
  28
  29
  30 def diff_index_filenames(ref):
  31     """Return a of filenames that have been modified relative to the index"""
  32     out = git.diff_index(ref, name_only=True, z=True)[STDOUT]
  33     return _parse_diff_filenames(out)
  34
  35
  36 def diff_filenames(*args):
  37     """Return a list of filenames that have been modified"""
  38     out = git.diff_tree(name_only=True, no_commit_id=True, r=True, z=True,
  39                         _readonly=True, *args)[STDOUT]
  40     return _parse_diff_filenames(out)
  41
  42
  43 def listdir(dirname, ref='HEAD'):
  44     """Get the contents of a directory according to Git
  45
  46     Query Git for the content of a directory, taking ignored
  47     files into account.
  48
  49     """
  50     dirs = []
  51     files = []
  52
  53     # first, parse git ls-tree to get the tracked files
  54     # in a list of (type, path) tuples
  55     entries = ls_tree(dirname, ref=ref)
  56     for entry in entries:
  57         if entry[0][0] == 't':  # tree
  58             dirs.append(entry[1])
  59         else:
  60             files.append(entry[1])
  61
  62     # gather untracked files
  63     untracked = untracked_files(paths=[dirname], directory=True)
  64     for path in untracked:
  65         if path.endswith('/'):
  66             dirs.append(path[:-1])
  67         else:
  68             files.append(path)
  69
  70     dirs.sort()
  71     files.sort()
  72
  73     return (dirs, files)
  74
  75
  76 def diff(args):
  77     """Return a list of filenames for the given diff arguments
  78
  79     :param args: list of arguments to pass to "git diff --name-only"
  80
  81     """
  82     out = git.diff(name_only=True, z=True, *args)[STDOUT]
  83     return _parse_diff_filenames(out)
  84
  85
  86 def _parse_diff_filenames(out):
  87     if out:
  88         return out[:-1].split('\0')
  89     else:
  90         return []
  91
  92
  93 def tracked_files(*args):
  94     """Return the names of all files in the repository"""
  95     out = git.ls_files('--', *args, z=True)[STDOUT]
  96     if out:
  97         return sorted(out[:-1].split('\0'))
  98     else:
  99         return []
 100
 101
 102 def all_files(*args):
 103     """Returns a sorted list of all files, including untracked files."""
 104     ls_files = git.ls_files('--', *args,
 105                             z=True,
 106                             cached=True,
 107                             others=True,
 108                             exclude_standard=True)[STDOUT]
 109     return sorted([f for f in ls_files.split('\0') if f])
 110
 111
 112 class _current_branch(object):
 113     """Cache for current_branch()"""
 114     key = None
 115     value = None
 116
 117
 118 def reset():
 119     _current_branch.key = None
 120
 121
 122 def current_branch():
 123     """Return the current branch"""
 124     head = git.git_path('HEAD')
 125     try:
 126         key = core.stat(head).st_mtime
 127         if _current_branch.key == key:
 128             return _current_branch.value
 129     except OSError:
 130         # OSError means we can't use the stat cache
 131         key = 0
 132
 133     status, data, err = git.rev_parse('HEAD', symbolic_full_name=True)
 134     if status != 0:
 135         # git init -- read .git/HEAD.  We could do this unconditionally...
 136         data = _read_git_head(head)
 137
 138     for refs_prefix in ('refs/heads/', 'refs/remotes/', 'refs/tags/'):
 139         if data.startswith(refs_prefix):
 140             value = data[len(refs_prefix):]
 141             _current_branch.key = key
 142             _current_branch.value = value
 143             return value
 144     # Detached head
 145     return data
 146
 147
 148 def _read_git_head(head, default='master', git=git):
 149     """Pure-python .git/HEAD reader"""
 150     # Common .git/HEAD "ref: refs/heads/master" files
 151     islink = core.islink(head)
 152     if core.isfile(head) and not islink:
 153         data = core.read(head).rstrip()
 154         ref_prefix = 'ref: '
 155         if data.startswith(ref_prefix):
 156             return data[len(ref_prefix):]
 157         # Detached head
 158         return data
 159     # Legacy .git/HEAD symlinks
 160     elif islink:
 161         refs_heads = core.realpath(git.git_path('refs', 'heads'))
 162         path = core.abspath(head).replace('\\', '/')
 163         if path.startswith(refs_heads + '/'):
 164             return path[len(refs_heads)+1:]
 165
 166     return default
 167
 168
 169 def branch_list(remote=False):
 170     """
 171     Return a list of local or remote branches
 172
 173     This explicitly removes HEAD from the list of remote branches.
 174
 175     """
 176     if remote:
 177         return for_each_ref_basename('refs/remotes')
 178     else:
 179         return for_each_ref_basename('refs/heads')
 180
 181
 182 def _version_sort():
 183     if version.check_git('version-sort'):
 184         sort = 'version:refname'
 185     else:
 186         sort = False
 187     return sort
 188
 189
 190 def for_each_ref_basename(refs, git=git):
 191     """Return refs starting with 'refs'."""
 192     sort = _version_sort()
 193     status, out, err = git.for_each_ref(refs, format='%(refname)',
 194                                         sort=sort, _readonly=True)
 195     output = out.splitlines()
 196     non_heads = [x for x in output if not x.endswith('/HEAD')]
 197     return list(map(lambda x: x[len(refs) + 1:], non_heads))
 198
 199
 200 def _triple(x, y):
 201     return (x, len(x) + 1, y)
 202
 203
 204 def all_refs(split=False, git=git):
 205     """Return a tuple of (local branches, remote branches, tags)."""
 206     local_branches = []
 207     remote_branches = []
 208     tags = []
 209     triple = _triple
 210     query = (triple('refs/tags', tags),
 211              triple('refs/heads', local_branches),
 212              triple('refs/remotes', remote_branches))
 213     sort = _version_sort()
 214     status, out, err = git.for_each_ref(format='%(refname)',
 215                                         sort=sort, _readonly=True)
 216     for ref in out.splitlines():
 217         for prefix, prefix_len, dst in query:
 218             if ref.startswith(prefix) and not ref.endswith('/HEAD'):
 219                 dst.append(ref[prefix_len:])
 220                 continue
 221     tags.reverse()
 222     if split:
 223         return local_branches, remote_branches, tags
 224     else:
 225         return local_branches + remote_branches + tags
 226
 227
 228 def tracked_branch(branch=None, config=None):
 229     """Return the remote branch associated with 'branch'."""
 230     if config is None:
 231         config = gitcfg.current()
 232     if branch is None:
 233         branch = current_branch()
 234     if branch is None:
 235         return None
 236     remote = config.get('branch.%s.remote' % branch)
 237     if not remote:
 238         return None
 239     merge_ref = config.get('branch.%s.merge' % branch)
 240     if not merge_ref:
 241         return None
 242     refs_heads = 'refs/heads/'
 243     if merge_ref.startswith(refs_heads):
 244         return remote + '/' + merge_ref[len(refs_heads):]
 245     return None
 246
 247
 248 def untracked_files(git=git, paths=None, **kwargs):
 249     """Returns a sorted list of untracked files."""
 250
 251     if paths is None:
 252         paths = []
 253     args = ['--'] + paths
 254     out = git.ls_files(z=True, others=True, exclude_standard=True,
 255                        *args, **kwargs)[STDOUT]
 256     if out:
 257         return out[:-1].split('\0')
 258     return []
 259
 260
 261 def tag_list():
 262     """Return a list of tags."""
 263     result = for_each_ref_basename('refs/tags')
 264     result.reverse()
 265     return result
 266
 267
 268 def log(git, *args, **kwargs):
 269     return git.log(no_color=True, no_abbrev_commit=True,
 270                    no_ext_diff=True, _readonly=True, *args, **kwargs)[STDOUT]
 271
 272
 273 def commit_diff(oid, git=git):
 274     return log(git, '-1', oid, '--') + '\n\n' + oid_diff(git, oid)
 275
 276
 277 _diff_overrides = {}
 278
 279
 280 def update_diff_overrides(space_at_eol, space_change,
 281                           all_space, function_context):
 282     _diff_overrides['ignore_space_at_eol'] = space_at_eol
 283     _diff_overrides['ignore_space_change'] = space_change
 284     _diff_overrides['ignore_all_space'] = all_space
 285     _diff_overrides['function_context'] = function_context
 286
 287
 288 def common_diff_opts(config=None):
 289     if config is None:
 290         config = gitcfg.current()
 291     submodule = version.check('diff-submodule', version.git_version())
 292     opts = {
 293         'patience': True,
 294         'submodule': submodule,
 295         'no_color': True,
 296         'no_ext_diff': True,
 297         'unified': config.get('gui.diffcontext', 3),
 298         '_raw': True,
 299     }
 300     opts.update(_diff_overrides)
 301     return opts
 302
 303
 304 def _add_filename(args, filename):
 305     if filename:
 306         args.extend(['--', filename])
 307
 308
 309 def oid_diff(git, oid, filename=None):
 310     """Return the diff for an oid"""
 311     # Naively "$oid^!" is what we'd like to use but that doesn't
 312     # give the correct result for merges--the diff is reversed.
 313     # Be explicit and compare oid against its first parent.
 314     args = [oid + '~', oid]
 315     opts = common_diff_opts()
 316     _add_filename(args, filename)
 317     status, out, err = git.diff(*args, **opts)
 318     if status != 0:
 319         # We probably don't have "$oid~" because this is the root commit.
 320         # "git show" is clever enough to handle the root commit.
 321         args = [oid + '^!']
 322         _add_filename(args, filename)
 323         status, out, err = git.show(pretty='format:', _readonly=True,
 324                                     *args, **opts)
 325         out = out.lstrip()
 326     return out
 327
 328
 329 def diff_info(oid, git=git, filename=None):
 330     decoded = log(git, '-1', oid, '--', pretty='format:%b').strip()
 331     if decoded:
 332         decoded += '\n\n'
 333     return decoded + oid_diff(git, oid, filename=filename)
 334
 335
 336 def diff_helper(commit=None,
 337                 ref=None,
 338                 endref=None,
 339                 filename=None,
 340                 cached=True,
 341                 deleted=False,
 342                 head=None,
 343                 amending=False,
 344                 with_diff_header=False,
 345                 suppress_header=True,
 346                 reverse=False,
 347                 git=git):
 348     "Invokes git diff on a filepath."
 349     if commit:
 350         ref, endref = commit+'^', commit
 351     argv = []
 352     if ref and endref:
 353         argv.append('%s..%s' % (ref, endref))
 354     elif ref:
 355         for r in utils.shell_split(ref.strip()):
 356             argv.append(r)
 357     elif head and amending and cached:
 358         argv.append(head)
 359
 360     encoding = None
 361     if filename:
 362         argv.append('--')
 363         if type(filename) is list:
 364             argv.extend(filename)
 365         else:
 366             argv.append(filename)
 367             cfg = gitcfg.current()
 368             encoding = cfg.file_encoding(filename)
 369
 370     status, out, err = git.diff(R=reverse, M=True, cached=cached,
 371                                 _encoding=encoding,
 372                                 *argv,
 373                                 **common_diff_opts())
 374     if status != 0:
 375         # git init
 376         if with_diff_header:
 377             return ('', '')
 378         else:
 379             return ''
 380
 381     return extract_diff_header(status, deleted,
 382                                with_diff_header, suppress_header, out)
 383
 384
 385 def extract_diff_header(status, deleted,
 386                         with_diff_header, suppress_header, diffoutput):
 387     headers = []
 388
 389     if diffoutput.startswith('Submodule'):
 390         if with_diff_header:
 391             return ('', diffoutput)
 392         else:
 393             return diffoutput
 394
 395     start = False
 396     del_tag = 'deleted file mode '
 397     output = StringIO()
 398
 399     for line in diffoutput.splitlines():
 400         if not start and '@@' == line[:2] and '@@' in line[2:]:
 401             start = True
 402         if start or (deleted and del_tag in line):
 403             output.write(line + '\n')
 404         else:
 405             if with_diff_header:
 406                 headers.append(line)
 407             elif not suppress_header:
 408                 output.write(line + '\n')
 409
 410     result = output.getvalue().rstrip('\n')
 411     output.close()
 412
 413     if with_diff_header:
 414         return('\n'.join(headers), result)
 415     else:
 416         return result
 417
 418
 419 def format_patchsets(to_export, revs, output='patches'):
 420     """
 421     Group contiguous revision selection into patchsets
 422
 423     Exists to handle multi-selection.
 424     Multiple disparate ranges in the revision selection
 425     are grouped into continuous lists.
 426
 427     """
 428
 429     outs = []
 430     errs = []
 431
 432     cur_rev = to_export[0]
 433     cur_master_idx = revs.index(cur_rev)
 434
 435     patches_to_export = [[cur_rev]]
 436     patchset_idx = 0
 437
 438     # Group the patches into continuous sets
 439     for idx, rev in enumerate(to_export[1:]):
 440         # Limit the search to the current neighborhood for efficiency
 441         try:
 442             master_idx = revs[cur_master_idx:].index(rev)
 443             master_idx += cur_master_idx
 444         except ValueError:
 445             master_idx = revs.index(rev)
 446
 447         if master_idx == cur_master_idx + 1:
 448             patches_to_export[patchset_idx].append(rev)
 449             cur_master_idx += 1
 450             continue
 451         else:
 452             patches_to_export.append([rev])
 453             cur_master_idx = master_idx
 454             patchset_idx += 1
 455
 456     # Export each patchsets
 457     status = 0
 458     for patchset in patches_to_export:
 459         stat, out, err = export_patchset(patchset[0],
 460                                          patchset[-1],
 461                                          output='patches',
 462                                          n=len(patchset) > 1,
 463                                          thread=True,
 464                                          patch_with_stat=True)
 465         outs.append(out)
 466         if err:
 467             errs.append(err)
 468         status = max(stat, status)
 469     return (status, '\n'.join(outs), '\n'.join(errs))
 470
 471
 472 def export_patchset(start, end, output='patches', **kwargs):
 473     """Export patches from start^ to end."""
 474     return git.format_patch('-o', output, start + '^..' + end, **kwargs)
 475
 476
 477 def unstage_paths(args, head='HEAD'):
 478     status, out, err = git.reset(head, '--', *set(args))
 479     if status == 128:
 480         # handle git init: we have to use 'git rm --cached'
 481         # detect this condition by checking if the file is still staged
 482         return untrack_paths(args, head=head)
 483     else:
 484         return (status, out, err)
 485
 486
 487 def untrack_paths(args, head='HEAD'):
 488     if not args:
 489         return (-1, N_('Nothing to do'), '')
 490     return git.update_index('--', force_remove=True, *set(args))
 491
 492
 493 def worktree_state(head='HEAD',
 494                    update_index=False,
 495                    display_untracked=True,
 496                    paths=None):
 497     """Return a dict of files in various states of being
 498
 499     :rtype: dict, keys are staged, unstaged, untracked, unmerged,
 500             changed_upstream, and submodule.
 501
 502     """
 503     if update_index:
 504         git.update_index(refresh=True)
 505
 506     staged, unmerged, staged_deleted, staged_submods = diff_index(head,
 507                                                                   paths=paths)
 508     modified, unstaged_deleted, modified_submods = diff_worktree(paths)
 509     untracked = display_untracked and untracked_files(paths=paths) or []
 510
 511     # Remove unmerged paths from the modified list
 512     if unmerged:
 513         unmerged_set = set(unmerged)
 514         modified = [path for path in modified if path not in unmerged_set]
 515
 516     # Look for upstream modified files if this is a tracking branch
 517     upstream_changed = diff_upstream(head)
 518
 519     # Keep stuff sorted
 520     staged.sort()
 521     modified.sort()
 522     unmerged.sort()
 523     untracked.sort()
 524     upstream_changed.sort()
 525
 526     return {'staged': staged,
 527             'modified': modified,
 528             'unmerged': unmerged,
 529             'untracked': untracked,
 530             'upstream_changed': upstream_changed,
 531             'staged_deleted': staged_deleted,
 532             'unstaged_deleted': unstaged_deleted,
 533             'submodules': staged_submods | modified_submods}
 534
 535
 536 def _parse_raw_diff(out):
 537     while out:
 538         info, path, out = out.split('\0', 2)
 539         status = info[-1]
 540         is_submodule = ('160000' in info[1:14])
 541         yield (path, status, is_submodule)
 542
 543
 544 def diff_index(head, cached=True, paths=None):
 545     staged = []
 546     unmerged = []
 547     deleted = set()
 548     submodules = set()
 549
 550     if paths is None:
 551         paths = []
 552     args = [head, '--'] + paths
 553     status, out, err = git.diff_index(cached=cached, z=True, *args)
 554     if status != 0:
 555         # handle git init
 556         args[0] = EMPTY_TREE_OID
 557         status, out, err = git.diff_index(cached=cached, z=True, *args)
 558
 559     for path, status, is_submodule in _parse_raw_diff(out):
 560         if is_submodule:
 561             submodules.add(path)
 562         if status in 'DAMT':
 563             staged.append(path)
 564             if status == 'D':
 565                 deleted.add(path)
 566         elif status == 'U':
 567             unmerged.append(path)
 568
 569     return staged, unmerged, deleted, submodules
 570
 571
 572 def diff_worktree(paths=None):
 573     modified = []
 574     deleted = set()
 575     submodules = set()
 576
 577     if paths is None:
 578         paths = []
 579     args = ['--'] + paths
 580     status, out, err = git.diff_files(z=True, *args)
 581     for path, status, is_submodule in _parse_raw_diff(out):
 582         if is_submodule:
 583             submodules.add(path)
 584         if status in 'DAMT':
 585             modified.append(path)
 586             if status == 'D':
 587                 deleted.add(path)
 588
 589     return modified, deleted, submodules
 590
 591
 592 def diff_upstream(head):
 593     tracked = tracked_branch()
 594     if not tracked:
 595         return []
 596     base = merge_base(head, tracked)
 597     return diff_filenames(base, tracked)
 598
 599
 600 def _branch_status(branch):
 601     """
 602     Returns a tuple of staged, unstaged, untracked, and unmerged files
 603
 604     This shows only the changes that were introduced in branch
 605
 606     """
 607     staged = diff_filenames(branch)
 608     return {'staged': staged,
 609             'upstream_changed': staged}
 610
 611
 612 def merge_base(head, ref):
 613     """Given `ref`, return $(git merge-base ref HEAD)..ref."""
 614     return git.merge_base(head, ref, _readonly=True)[STDOUT]
 615
 616
 617 def merge_base_parent(branch):
 618     tracked = tracked_branch(branch=branch)
 619     if tracked:
 620         return tracked
 621     return 'HEAD'
 622
 623
 624 def parse_ls_tree(rev):
 625     """Return a list of (mode, type, oid, path) tuples."""
 626     output = []
 627     lines = git.ls_tree(rev, r=True, _readonly=True)[STDOUT].splitlines()
 628     regex = re.compile(r'^(\d+)\W(\w+)\W(\w+)[ \t]+(.*)$')
 629     for line in lines:
 630         match = regex.match(line)
 631         if match:
 632             mode = match.group(1)
 633             objtype = match.group(2)
 634             oid = match.group(3)
 635             filename = match.group(4)
 636             output.append((mode, objtype, oid, filename,))
 637     return output
 638
 639
 640 def ls_tree(path, ref='HEAD'):
 641     """Return a parsed git ls-tree result for a single directory"""
 642
 643     result = []
 644     status, out, err = git.ls_tree(ref, '--', path, z=True, full_tree=True)
 645     if status == 0 and out:
 646         for line in out[:-1].split('\0'):
 647             # .....6 ...4 ......................................40
 648             # 040000 tree c127cde9a0c644a3a8fef449a244f47d5272dfa6      relative
 649             # 100644 blob 139e42bf4acaa4927ec9be1ec55a252b97d3f1e2      relative/path
 650             # 0..... 7... 12......................................      53
 651             # path offset = 6 + 1 + 4 + 1 + 40 + 1 = 53
 652             objtype = line[7:11]
 653             relpath = line[53:]
 654             result.append((objtype, relpath))
 655
 656     return result
 657
 658 # A regex for matching the output of git(log|rev-list) --pretty=oneline
 659 REV_LIST_REGEX = re.compile(r'^([0-9a-f]{40}) (.*)$')
 660
 661
 662 def parse_rev_list(raw_revs):
 663     """Parse `git log --pretty=online` output into (oid, summary) pairs."""
 664     revs = []
 665     for line in raw_revs.splitlines():
 666         match = REV_LIST_REGEX.match(line)
 667         if match:
 668             rev_id = match.group(1)
 669             summary = match.group(2)
 670             revs.append((rev_id, summary,))
 671     return revs
 672
 673
 674 def log_helper(all=False, extra_args=None):
 675     """Return parallel arrays containing oids and summaries."""
 676     revs = []
 677     summaries = []
 678     args = []
 679     if extra_args:
 680         args = extra_args
 681     output = log(git, pretty='oneline', all=all, *args)
 682     for line in output.splitlines():
 683         match = REV_LIST_REGEX.match(line)
 684         if match:
 685             revs.append(match.group(1))
 686             summaries.append(match.group(2))
 687     return (revs, summaries)
 688
 689
 690 def rev_list_range(start, end):
 691     """Return (oid, summary) pairs between start and end."""
 692     revrange = '%s..%s' % (start, end)
 693     out = git.rev_list(revrange, pretty='oneline')[STDOUT]
 694     return parse_rev_list(out)
 695
 696
 697 def commit_message_path():
 698     """Return the path to .git/GIT_COLA_MSG"""
 699     path = git.git_path('GIT_COLA_MSG')
 700     if core.exists(path):
 701         return path
 702     return None
 703
 704
 705 def merge_message_path():
 706     """Return the path to .git/MERGE_MSG or .git/SQUASH_MSG."""
 707     for basename in ('MERGE_MSG', 'SQUASH_MSG'):
 708         path = git.git_path(basename)
 709         if core.exists(path):
 710             return path
 711     return None
 712
 713
 714 def prepare_commit_message_hook(config=None):
 715     default_hook = git.git_path('hooks', 'cola-prepare-commit-msg')
 716     if config is None:
 717         config = gitcfg.current()
 718     return config.get('cola.preparecommitmessagehook', default_hook)
 719
 720
 721 def abort_merge():
 722     """Abort a merge by reading the tree at HEAD."""
 723     # Reset the worktree
 724     git.read_tree('HEAD', reset=True, u=True, v=True)
 725     # remove MERGE_HEAD
 726     merge_head = git.git_path('MERGE_HEAD')
 727     if core.exists(merge_head):
 728         core.unlink(merge_head)
 729     # remove MERGE_MESSAGE, etc.
 730     merge_msg_path = merge_message_path()
 731     while merge_msg_path:
 732         core.unlink(merge_msg_path)
 733         merge_msg_path = merge_message_path()
 734
 735
 736 def strip_remote(remotes, remote_branch):
 737     for remote in remotes:
 738         prefix = remote + '/'
 739         if remote_branch.startswith(prefix):
 740             return remote_branch[len(prefix):]
 741     return remote_branch.split('/', 1)[-1]