cola/gitcmds.py

   1 """Provides commands and queries for Git."""
   2 from __future__ import division, absolute_import, unicode_literals
   3
   4 import re
   5 from io import StringIO
   6
   7 from . import core
   8 from . import gitcfg
   9 from . import utils
  10 from . import version
  11 from .git import git
  12 from .git import STDOUT
  13 from .i18n import N_
  14
  15
  16 EMPTY_TREE_OID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'
  17
  18
  19 class InvalidRepositoryError(Exception):
  20     pass
  21
  22
  23 def default_remote(config=None):
  24     """Return the remote tracked by the current branch."""
  25     if config is None:
  26         config = gitcfg.current()
  27     return config.get('branch.%s.remote' % current_branch())
  28
  29
  30 def diff_index_filenames(ref):
  31     """Return a of filenames that have been modified relative to the index"""
  32     out = git.diff_index(ref, name_only=True, z=True)[STDOUT]
  33     return _parse_diff_filenames(out)
  34
  35
  36 def diff_filenames(*args):
  37     """Return a list of filenames that have been modified"""
  38     out = git.diff_tree(name_only=True, no_commit_id=True, r=True, z=True,
  39                         _readonly=True, *args)[STDOUT]
  40     return _parse_diff_filenames(out)
  41
  42
  43 def listdir(dirname, ref='HEAD'):
  44     """Get the contents of a directory according to Git
  45
  46     Query Git for the content of a directory, taking ignored
  47     files into account.
  48
  49     """
  50     dirs = []
  51     files = []
  52
  53     # first, parse git ls-tree to get the tracked files
  54     # in a list of (type, path) tuples
  55     entries = ls_tree(dirname, ref=ref)
  56     for entry in entries:
  57         if entry[0][0] == 't':  # tree
  58             dirs.append(entry[1])
  59         else:
  60             files.append(entry[1])
  61
  62     # gather untracked files
  63     untracked = untracked_files(paths=[dirname], directory=True)
  64     for path in untracked:
  65         if path.endswith('/'):
  66             dirs.append(path[:-1])
  67         else:
  68             files.append(path)
  69
  70     dirs.sort()
  71     files.sort()
  72
  73     return (dirs, files)
  74
  75
  76 def diff(args):
  77     """Return a list of filenames for the given diff arguments
  78
  79     :param args: list of arguments to pass to "git diff --name-only"
  80
  81     """
  82     out = git.diff(name_only=True, z=True, *args)[STDOUT]
  83     return _parse_diff_filenames(out)
  84
  85
  86 def _parse_diff_filenames(out):
  87     if out:
  88         return out[:-1].split('\0')
  89     else:
  90         return []
  91
  92
  93 def tracked_files(*args):
  94     """Return the names of all files in the repository"""
  95     out = git.ls_files('--', *args, z=True)[STDOUT]
  96     if out:
  97         return sorted(out[:-1].split('\0'))
  98     else:
  99         return []
 100
 101
 102 def all_files(*args):
 103     """Returns a sorted list of all files, including untracked files."""
 104     ls_files = git.ls_files('--', *args,
 105                             z=True,
 106                             cached=True,
 107                             others=True,
 108                             exclude_standard=True)[STDOUT]
 109     return sorted([f for f in ls_files.split('\0') if f])
 110
 111
 112 class _current_branch(object):
 113     """Cache for current_branch()"""
 114     key = None
 115     value = None
 116
 117
 118 def reset():
 119     _current_branch.key = None
 120
 121
 122 def current_branch():
 123     """Return the current branch"""
 124     head = git.git_path('HEAD')
 125     try:
 126         key = core.stat(head).st_mtime
 127         if _current_branch.key == key:
 128             return _current_branch.value
 129     except OSError:
 130         # OSError means we can't use the stat cache
 131         key = 0
 132
 133     status, data, err = git.rev_parse('HEAD', symbolic_full_name=True)
 134     if status != 0:
 135         # git init -- read .git/HEAD.  We could do this unconditionally...
 136         data = _read_git_head(head)
 137
 138     for refs_prefix in ('refs/heads/', 'refs/remotes/', 'refs/tags/'):
 139         if data.startswith(refs_prefix):
 140             value = data[len(refs_prefix):]
 141             _current_branch.key = key
 142             _current_branch.value = value
 143             return value
 144     # Detached head
 145     return data
 146
 147
 148 def _read_git_head(head, default='master', git=git):
 149     """Pure-python .git/HEAD reader"""
 150     # Common .git/HEAD "ref: refs/heads/master" files
 151     islink = core.islink(head)
 152     if core.isfile(head) and not islink:
 153         data = core.read(head).rstrip()
 154         ref_prefix = 'ref: '
 155         if data.startswith(ref_prefix):
 156             return data[len(ref_prefix):]
 157         # Detached head
 158         return data
 159     # Legacy .git/HEAD symlinks
 160     elif islink:
 161         refs_heads = core.realpath(git.git_path('refs', 'heads'))
 162         path = core.abspath(head).replace('\\', '/')
 163         if path.startswith(refs_heads + '/'):
 164             return path[len(refs_heads)+1:]
 165
 166     return default
 167
 168
 169 def branch_list(remote=False):
 170     """
 171     Return a list of local or remote branches
 172
 173     This explicitly removes HEAD from the list of remote branches.
 174
 175     """
 176     if remote:
 177         return for_each_ref_basename('refs/remotes')
 178     else:
 179         return for_each_ref_basename('refs/heads')
 180
 181
 182 def for_each_ref_basename(refs, git=git):
 183     """Return refs starting with 'refs'."""
 184     out = git.for_each_ref(refs, format='%(refname)', _readonly=True)[STDOUT]
 185     output = out.splitlines()
 186     non_heads = [x for x in output if not x.endswith('/HEAD')]
 187     return list(map(lambda x: x[len(refs) + 1:], non_heads))
 188
 189
 190 def _triple(x, y):
 191     return (x, len(x) + 1, y)
 192
 193
 194 def all_refs(split=False, git=git):
 195     """Return a tuple of (local branches, remote branches, tags)."""
 196     local_branches = []
 197     remote_branches = []
 198     tags = []
 199     triple = _triple
 200     query = (triple('refs/tags', tags),
 201              triple('refs/heads', local_branches),
 202              triple('refs/remotes', remote_branches))
 203     out = git.for_each_ref(format='%(refname)', _readonly=True)[STDOUT]
 204     for ref in out.splitlines():
 205         for prefix, prefix_len, dst in query:
 206             if ref.startswith(prefix) and not ref.endswith('/HEAD'):
 207                 dst.append(ref[prefix_len:])
 208                 continue
 209     if split:
 210         return local_branches, remote_branches, tags
 211     else:
 212         return local_branches + remote_branches + tags
 213
 214
 215 def tracked_branch(branch=None, config=None):
 216     """Return the remote branch associated with 'branch'."""
 217     if config is None:
 218         config = gitcfg.current()
 219     if branch is None:
 220         branch = current_branch()
 221     if branch is None:
 222         return None
 223     remote = config.get('branch.%s.remote' % branch)
 224     if not remote:
 225         return None
 226     merge_ref = config.get('branch.%s.merge' % branch)
 227     if not merge_ref:
 228         return None
 229     refs_heads = 'refs/heads/'
 230     if merge_ref.startswith(refs_heads):
 231         return remote + '/' + merge_ref[len(refs_heads):]
 232     return None
 233
 234
 235 def untracked_files(git=git, paths=None, **kwargs):
 236     """Returns a sorted list of untracked files."""
 237
 238     if paths is None:
 239         paths = []
 240     args = ['--'] + paths
 241     out = git.ls_files(z=True, others=True, exclude_standard=True,
 242                        *args, **kwargs)[STDOUT]
 243     if out:
 244         return out[:-1].split('\0')
 245     return []
 246
 247
 248 def tag_list():
 249     """Return a list of tags."""
 250     return list(reversed(for_each_ref_basename('refs/tags')))
 251
 252
 253 def log(git, *args, **kwargs):
 254     return git.log(no_color=True, no_abbrev_commit=True,
 255                    no_ext_diff=True, _readonly=True, *args, **kwargs)[STDOUT]
 256
 257
 258 def commit_diff(oid, git=git):
 259     return log(git, '-1', oid, '--') + '\n\n' + oid_diff(git, oid)
 260
 261
 262 _diff_overrides = {}
 263
 264
 265 def update_diff_overrides(space_at_eol, space_change,
 266                           all_space, function_context):
 267     _diff_overrides['ignore_space_at_eol'] = space_at_eol
 268     _diff_overrides['ignore_space_change'] = space_change
 269     _diff_overrides['ignore_all_space'] = all_space
 270     _diff_overrides['function_context'] = function_context
 271
 272
 273 def common_diff_opts(config=None):
 274     if config is None:
 275         config = gitcfg.current()
 276     submodule = version.check('diff-submodule', version.git_version())
 277     opts = {
 278         'patience': True,
 279         'submodule': submodule,
 280         'no_color': True,
 281         'no_ext_diff': True,
 282         'unified': config.get('gui.diffcontext', 3),
 283         '_raw': True,
 284     }
 285     opts.update(_diff_overrides)
 286     return opts
 287
 288
 289 def _add_filename(args, filename):
 290     if filename:
 291         args.extend(['--', filename])
 292
 293
 294 def oid_diff(git, oid, filename=None):
 295     """Return the diff for an oid"""
 296     # Naively "$oid^!" is what we'd like to use but that doesn't
 297     # give the correct result for merges--the diff is reversed.
 298     # Be explicit and compare oid against its first parent.
 299     args = [oid + '~', oid]
 300     opts = common_diff_opts()
 301     _add_filename(args, filename)
 302     status, out, err = git.diff(*args, **opts)
 303     if status != 0:
 304         # We probably don't have "$oid~" because this is the root commit.
 305         # "git show" is clever enough to handle the root commit.
 306         args = [oid + '^!']
 307         _add_filename(args, filename)
 308         status, out, err = git.show(pretty='format:', _readonly=True,
 309                                     *args, **opts)
 310         out = out.lstrip()
 311     return out
 312
 313
 314 def diff_info(oid, git=git, filename=None):
 315     decoded = log(git, '-1', oid, '--', pretty='format:%b').strip()
 316     if decoded:
 317         decoded += '\n\n'
 318     return decoded + oid_diff(git, oid, filename=filename)
 319
 320
 321 def diff_helper(commit=None,
 322                 ref=None,
 323                 endref=None,
 324                 filename=None,
 325                 cached=True,
 326                 deleted=False,
 327                 head=None,
 328                 amending=False,
 329                 with_diff_header=False,
 330                 suppress_header=True,
 331                 reverse=False,
 332                 git=git):
 333     "Invokes git diff on a filepath."
 334     if commit:
 335         ref, endref = commit+'^', commit
 336     argv = []
 337     if ref and endref:
 338         argv.append('%s..%s' % (ref, endref))
 339     elif ref:
 340         for r in utils.shell_split(ref.strip()):
 341             argv.append(r)
 342     elif head and amending and cached:
 343         argv.append(head)
 344
 345     encoding = None
 346     if filename:
 347         argv.append('--')
 348         if type(filename) is list:
 349             argv.extend(filename)
 350         else:
 351             argv.append(filename)
 352             cfg = gitcfg.current()
 353             encoding = cfg.file_encoding(filename)
 354
 355     status, out, err = git.diff(R=reverse, M=True, cached=cached,
 356                                 _encoding=encoding,
 357                                 *argv,
 358                                 **common_diff_opts())
 359     if status != 0:
 360         # git init
 361         if with_diff_header:
 362             return ('', '')
 363         else:
 364             return ''
 365
 366     return extract_diff_header(status, deleted,
 367                                with_diff_header, suppress_header, out)
 368
 369
 370 def extract_diff_header(status, deleted,
 371                         with_diff_header, suppress_header, diffoutput):
 372     headers = []
 373
 374     if diffoutput.startswith('Submodule'):
 375         if with_diff_header:
 376             return ('', diffoutput)
 377         else:
 378             return diffoutput
 379
 380     start = False
 381     del_tag = 'deleted file mode '
 382     output = StringIO()
 383
 384     for line in diffoutput.splitlines():
 385         if not start and '@@' == line[:2] and '@@' in line[2:]:
 386             start = True
 387         if start or (deleted and del_tag in line):
 388             output.write(line + '\n')
 389         else:
 390             if with_diff_header:
 391                 headers.append(line)
 392             elif not suppress_header:
 393                 output.write(line + '\n')
 394
 395     result = output.getvalue().rstrip('\n')
 396     output.close()
 397
 398     if with_diff_header:
 399         return('\n'.join(headers), result)
 400     else:
 401         return result
 402
 403
 404 def format_patchsets(to_export, revs, output='patches'):
 405     """
 406     Group contiguous revision selection into patchsets
 407
 408     Exists to handle multi-selection.
 409     Multiple disparate ranges in the revision selection
 410     are grouped into continuous lists.
 411
 412     """
 413
 414     outs = []
 415     errs = []
 416
 417     cur_rev = to_export[0]
 418     cur_master_idx = revs.index(cur_rev)
 419
 420     patches_to_export = [[cur_rev]]
 421     patchset_idx = 0
 422
 423     # Group the patches into continuous sets
 424     for idx, rev in enumerate(to_export[1:]):
 425         # Limit the search to the current neighborhood for efficiency
 426         try:
 427             master_idx = revs[cur_master_idx:].index(rev)
 428             master_idx += cur_master_idx
 429         except ValueError:
 430             master_idx = revs.index(rev)
 431
 432         if master_idx == cur_master_idx + 1:
 433             patches_to_export[patchset_idx].append(rev)
 434             cur_master_idx += 1
 435             continue
 436         else:
 437             patches_to_export.append([rev])
 438             cur_master_idx = master_idx
 439             patchset_idx += 1
 440
 441     # Export each patchsets
 442     status = 0
 443     for patchset in patches_to_export:
 444         stat, out, err = export_patchset(patchset[0],
 445                                          patchset[-1],
 446                                          output='patches',
 447                                          n=len(patchset) > 1,
 448                                          thread=True,
 449                                          patch_with_stat=True)
 450         outs.append(out)
 451         if err:
 452             errs.append(err)
 453         status = max(stat, status)
 454     return (status, '\n'.join(outs), '\n'.join(errs))
 455
 456
 457 def export_patchset(start, end, output='patches', **kwargs):
 458     """Export patches from start^ to end."""
 459     return git.format_patch('-o', output, start + '^..' + end, **kwargs)
 460
 461
 462 def unstage_paths(args, head='HEAD'):
 463     status, out, err = git.reset(head, '--', *set(args))
 464     if status == 128:
 465         # handle git init: we have to use 'git rm --cached'
 466         # detect this condition by checking if the file is still staged
 467         return untrack_paths(args, head=head)
 468     else:
 469         return (status, out, err)
 470
 471
 472 def untrack_paths(args, head='HEAD'):
 473     if not args:
 474         return (-1, N_('Nothing to do'), '')
 475     return git.update_index('--', force_remove=True, *set(args))
 476
 477
 478 def worktree_state(head='HEAD',
 479                    update_index=False,
 480                    display_untracked=True,
 481                    paths=None):
 482     """Return a dict of files in various states of being
 483
 484     :rtype: dict, keys are staged, unstaged, untracked, unmerged,
 485             changed_upstream, and submodule.
 486
 487     """
 488     if update_index:
 489         git.update_index(refresh=True)
 490
 491     staged, unmerged, staged_deleted, staged_submods = diff_index(head,
 492                                                                   paths=paths)
 493     modified, unstaged_deleted, modified_submods = diff_worktree(paths)
 494     untracked = display_untracked and untracked_files(paths=paths) or []
 495
 496     # Remove unmerged paths from the modified list
 497     if unmerged:
 498         unmerged_set = set(unmerged)
 499         modified = [path for path in modified if path not in unmerged_set]
 500
 501     # Look for upstream modified files if this is a tracking branch
 502     upstream_changed = diff_upstream(head)
 503
 504     # Keep stuff sorted
 505     staged.sort()
 506     modified.sort()
 507     unmerged.sort()
 508     untracked.sort()
 509     upstream_changed.sort()
 510
 511     return {'staged': staged,
 512             'modified': modified,
 513             'unmerged': unmerged,
 514             'untracked': untracked,
 515             'upstream_changed': upstream_changed,
 516             'staged_deleted': staged_deleted,
 517             'unstaged_deleted': unstaged_deleted,
 518             'submodules': staged_submods | modified_submods}
 519
 520
 521 def _parse_raw_diff(out):
 522     while out:
 523         info, path, out = out.split('\0', 2)
 524         status = info[-1]
 525         is_submodule = ('160000' in info[1:14])
 526         yield (path, status, is_submodule)
 527
 528
 529 def diff_index(head, cached=True, paths=None):
 530     staged = []
 531     unmerged = []
 532     deleted = set()
 533     submodules = set()
 534
 535     if paths is None:
 536         paths = []
 537     args = [head, '--'] + paths
 538     status, out, err = git.diff_index(cached=cached, z=True, *args)
 539     if status != 0:
 540         # handle git init
 541         args[0] = EMPTY_TREE_OID
 542         status, out, err = git.diff_index(cached=cached, z=True, *args)
 543
 544     for path, status, is_submodule in _parse_raw_diff(out):
 545         if is_submodule:
 546             submodules.add(path)
 547         if status in 'DAMT':
 548             staged.append(path)
 549             if status == 'D':
 550                 deleted.add(path)
 551         elif status == 'U':
 552             unmerged.append(path)
 553
 554     return staged, unmerged, deleted, submodules
 555
 556
 557 def diff_worktree(paths=None):
 558     modified = []
 559     deleted = set()
 560     submodules = set()
 561
 562     if paths is None:
 563         paths = []
 564     args = ['--'] + paths
 565     status, out, err = git.diff_files(z=True, *args)
 566     for path, status, is_submodule in _parse_raw_diff(out):
 567         if is_submodule:
 568             submodules.add(path)
 569         if status in 'DAMT':
 570             modified.append(path)
 571             if status == 'D':
 572                 deleted.add(path)
 573
 574     return modified, deleted, submodules
 575
 576
 577 def diff_upstream(head):
 578     tracked = tracked_branch()
 579     if not tracked:
 580         return []
 581     base = merge_base(head, tracked)
 582     return diff_filenames(base, tracked)
 583
 584
 585 def _branch_status(branch):
 586     """
 587     Returns a tuple of staged, unstaged, untracked, and unmerged files
 588
 589     This shows only the changes that were introduced in branch
 590
 591     """
 592     staged = diff_filenames(branch)
 593     return {'staged': staged,
 594             'upstream_changed': staged}
 595
 596
 597 def merge_base(head, ref):
 598     """Given `ref`, return $(git merge-base ref HEAD)..ref."""
 599     return git.merge_base(head, ref, _readonly=True)[STDOUT]
 600
 601
 602 def merge_base_parent(branch):
 603     tracked = tracked_branch(branch=branch)
 604     if tracked:
 605         return tracked
 606     return 'HEAD'
 607
 608
 609 def parse_ls_tree(rev):
 610     """Return a list of (mode, type, oid, path) tuples."""
 611     output = []
 612     lines = git.ls_tree(rev, r=True, _readonly=True)[STDOUT].splitlines()
 613     regex = re.compile(r'^(\d+)\W(\w+)\W(\w+)[ \t]+(.*)$')
 614     for line in lines:
 615         match = regex.match(line)
 616         if match:
 617             mode = match.group(1)
 618             objtype = match.group(2)
 619             oid = match.group(3)
 620             filename = match.group(4)
 621             output.append((mode, objtype, oid, filename,))
 622     return output
 623
 624
 625 def ls_tree(path, ref='HEAD'):
 626     """Return a parsed git ls-tree result for a single directory"""
 627
 628     result = []
 629     status, out, err = git.ls_tree(ref, '--', path, z=True, full_tree=True)
 630     if status == 0 and out:
 631         for line in out[:-1].split('\0'):
 632             # .....6 ...4 ......................................40
 633             # 040000 tree c127cde9a0c644a3a8fef449a244f47d5272dfa6      relative
 634             # 100644 blob 139e42bf4acaa4927ec9be1ec55a252b97d3f1e2      relative/path
 635             # 0..... 7... 12......................................      53
 636             # path offset = 6 + 1 + 4 + 1 + 40 + 1 = 53
 637             objtype = line[7:11]
 638             relpath = line[53:]
 639             result.append((objtype, relpath))
 640
 641     return result
 642
 643 # A regex for matching the output of git(log|rev-list) --pretty=oneline
 644 REV_LIST_REGEX = re.compile(r'^([0-9a-f]{40}) (.*)$')
 645
 646
 647 def parse_rev_list(raw_revs):
 648     """Parse `git log --pretty=online` output into (oid, summary) pairs."""
 649     revs = []
 650     for line in raw_revs.splitlines():
 651         match = REV_LIST_REGEX.match(line)
 652         if match:
 653             rev_id = match.group(1)
 654             summary = match.group(2)
 655             revs.append((rev_id, summary,))
 656     return revs
 657
 658
 659 def log_helper(all=False, extra_args=None):
 660     """Return parallel arrays containing oids and summaries."""
 661     revs = []
 662     summaries = []
 663     args = []
 664     if extra_args:
 665         args = extra_args
 666     output = log(git, pretty='oneline', all=all, *args)
 667     for line in output.splitlines():
 668         match = REV_LIST_REGEX.match(line)
 669         if match:
 670             revs.append(match.group(1))
 671             summaries.append(match.group(2))
 672     return (revs, summaries)
 673
 674
 675 def rev_list_range(start, end):
 676     """Return (oid, summary) pairs between start and end."""
 677     revrange = '%s..%s' % (start, end)
 678     out = git.rev_list(revrange, pretty='oneline')[STDOUT]
 679     return parse_rev_list(out)
 680
 681
 682 def commit_message_path():
 683     """Return the path to .git/GIT_COLA_MSG"""
 684     path = git.git_path('GIT_COLA_MSG')
 685     if core.exists(path):
 686         return path
 687     return None
 688
 689
 690 def merge_message_path():
 691     """Return the path to .git/MERGE_MSG or .git/SQUASH_MSG."""
 692     for basename in ('MERGE_MSG', 'SQUASH_MSG'):
 693         path = git.git_path(basename)
 694         if core.exists(path):
 695             return path
 696     return None
 697
 698
 699 def prepare_commit_message_hook(config=None):
 700     default_hook = git.git_path('hooks', 'cola-prepare-commit-msg')
 701     if config is None:
 702         config = gitcfg.current()
 703     return config.get('cola.preparecommitmessagehook', default_hook)
 704
 705
 706 def abort_merge():
 707     """Abort a merge by reading the tree at HEAD."""
 708     # Reset the worktree
 709     git.read_tree('HEAD', reset=True, u=True, v=True)
 710     # remove MERGE_HEAD
 711     merge_head = git.git_path('MERGE_HEAD')
 712     if core.exists(merge_head):
 713         core.unlink(merge_head)
 714     # remove MERGE_MESSAGE, etc.
 715     merge_msg_path = merge_message_path()
 716     while merge_msg_path:
 717         core.unlink(merge_msg_path)
 718         merge_msg_path = merge_message_path()
 719
 720
 721 def strip_remote(remotes, remote_branch):
 722     for remote in remotes:
 723         prefix = remote + '/'
 724         if remote_branch.startswith(prefix):
 725             return remote_branch[len(prefix):]
 726     return remote_branch.split('/', 1)[-1]