contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = ["p4"]
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += ["-u",user]
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += ["-P", password]
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += ["-p", port]
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += ["-h", host]
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += ["-c", client]
  46
  47
  48     if isinstance(cmd,basestring):
  49         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50     else:
  51         real_cmd += cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56     # not using the shell, we have to set it ourselves.  This path could
  57     # be relative, so go there first, then figure out where we ended up.
  58     os.chdir(dir)
  59     os.environ['PWD'] = os.getcwd()
  60
  61 def die(msg):
  62     if verbose:
  63         raise Exception(msg)
  64     else:
  65         sys.stderr.write(msg + "\n")
  66         sys.exit(1)
  67
  68 def write_pipe(c, stdin):
  69     if verbose:
  70         sys.stderr.write('Writing pipe: %s\n' % str(c))
  71
  72     expand = isinstance(c,basestring)
  73     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  74     pipe = p.stdin
  75     val = pipe.write(stdin)
  76     pipe.close()
  77     if p.wait():
  78         die('Command failed: %s' % str(c))
  79
  80     return val
  81
  82 def p4_write_pipe(c, stdin):
  83     real_cmd = p4_build_cmd(c)
  84     return write_pipe(real_cmd, stdin)
  85
  86 def read_pipe(c, ignore_error=False):
  87     if verbose:
  88         sys.stderr.write('Reading pipe: %s\n' % str(c))
  89
  90     expand = isinstance(c,basestring)
  91     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  92     pipe = p.stdout
  93     val = pipe.read()
  94     if p.wait() and not ignore_error:
  95         die('Command failed: %s' % str(c))
  96
  97     return val
  98
  99 def p4_read_pipe(c, ignore_error=False):
 100     real_cmd = p4_build_cmd(c)
 101     return read_pipe(real_cmd, ignore_error)
 102
 103 def read_pipe_lines(c):
 104     if verbose:
 105         sys.stderr.write('Reading pipe: %s\n' % str(c))
 106
 107     expand = isinstance(c, basestring)
 108     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 109     pipe = p.stdout
 110     val = pipe.readlines()
 111     if pipe.close() or p.wait():
 112         die('Command failed: %s' % str(c))
 113
 114     return val
 115
 116 def p4_read_pipe_lines(c):
 117     """Specifically invoke p4 on the command supplied. """
 118     real_cmd = p4_build_cmd(c)
 119     return read_pipe_lines(real_cmd)
 120
 121 def system(cmd):
 122     expand = isinstance(cmd,basestring)
 123     if verbose:
 124         sys.stderr.write("executing %s\n" % str(cmd))
 125     subprocess.check_call(cmd, shell=expand)
 126
 127 def p4_system(cmd):
 128     """Specifically invoke p4 as the system command. """
 129     real_cmd = p4_build_cmd(cmd)
 130     expand = isinstance(real_cmd, basestring)
 131     subprocess.check_call(real_cmd, shell=expand)
 132
 133 def p4_integrate(src, dest):
 134     p4_system(["integrate", "-Dt", src, dest])
 135
 136 def p4_sync(path):
 137     p4_system(["sync", path])
 138
 139 def p4_add(f):
 140     p4_system(["add", f])
 141
 142 def p4_delete(f):
 143     p4_system(["delete", f])
 144
 145 def p4_edit(f):
 146     p4_system(["edit", f])
 147
 148 def p4_revert(f):
 149     p4_system(["revert", f])
 150
 151 def p4_reopen(type, file):
 152     p4_system(["reopen", "-t", type, file])
 153
 154 #
 155 # Canonicalize the p4 type and return a tuple of the
 156 # base type, plus any modifiers.  See "p4 help filetypes"
 157 # for a list and explanation.
 158 #
 159 def split_p4_type(p4type):
 160
 161     p4_filetypes_historical = {
 162         "ctempobj": "binary+Sw",
 163         "ctext": "text+C",
 164         "cxtext": "text+Cx",
 165         "ktext": "text+k",
 166         "kxtext": "text+kx",
 167         "ltext": "text+F",
 168         "tempobj": "binary+FSw",
 169         "ubinary": "binary+F",
 170         "uresource": "resource+F",
 171         "uxbinary": "binary+Fx",
 172         "xbinary": "binary+x",
 173         "xltext": "text+Fx",
 174         "xtempobj": "binary+Swx",
 175         "xtext": "text+x",
 176         "xunicode": "unicode+x",
 177         "xutf16": "utf16+x",
 178     }
 179     if p4type in p4_filetypes_historical:
 180         p4type = p4_filetypes_historical[p4type]
 181     mods = ""
 182     s = p4type.split("+")
 183     base = s[0]
 184     mods = ""
 185     if len(s) > 1:
 186         mods = s[1]
 187     return (base, mods)
 188
 189
 190 def setP4ExecBit(file, mode):
 191     # Reopens an already open file and changes the execute bit to match
 192     # the execute bit setting in the passed in mode.
 193
 194     p4Type = "+x"
 195
 196     if not isModeExec(mode):
 197         p4Type = getP4OpenedType(file)
 198         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 199         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 200         if p4Type[-1] == "+":
 201             p4Type = p4Type[0:-1]
 202
 203     p4_reopen(p4Type, file)
 204
 205 def getP4OpenedType(file):
 206     # Returns the perforce file type for the given file.
 207
 208     result = p4_read_pipe(["opened", file])
 209     match = re.match(".*\((.+)\)\r?$", result)
 210     if match:
 211         return match.group(1)
 212     else:
 213         die("Could not determine file type for %s (result: '%s')" % (file, result))
 214
 215 def diffTreePattern():
 216     # This is a simple generator for the diff tree regex pattern. This could be
 217     # a class variable if this and parseDiffTreeEntry were a part of a class.
 218     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 219     while True:
 220         yield pattern
 221
 222 def parseDiffTreeEntry(entry):
 223     """Parses a single diff tree entry into its component elements.
 224
 225     See git-diff-tree(1) manpage for details about the format of the diff
 226     output. This method returns a dictionary with the following elements:
 227
 228     src_mode - The mode of the source file
 229     dst_mode - The mode of the destination file
 230     src_sha1 - The sha1 for the source file
 231     dst_sha1 - The sha1 fr the destination file
 232     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 233     status_score - The score for the status (applicable for 'C' and 'R'
 234                    statuses). This is None if there is no score.
 235     src - The path for the source file.
 236     dst - The path for the destination file. This is only present for
 237           copy or renames. If it is not present, this is None.
 238
 239     If the pattern is not matched, None is returned."""
 240
 241     match = diffTreePattern().next().match(entry)
 242     if match:
 243         return {
 244             'src_mode': match.group(1),
 245             'dst_mode': match.group(2),
 246             'src_sha1': match.group(3),
 247             'dst_sha1': match.group(4),
 248             'status': match.group(5),
 249             'status_score': match.group(6),
 250             'src': match.group(7),
 251             'dst': match.group(10)
 252         }
 253     return None
 254
 255 def isModeExec(mode):
 256     # Returns True if the given git mode represents an executable file,
 257     # otherwise False.
 258     return mode[-3:] == "755"
 259
 260 def isModeExecChanged(src_mode, dst_mode):
 261     return isModeExec(src_mode) != isModeExec(dst_mode)
 262
 263 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 264
 265     if isinstance(cmd,basestring):
 266         cmd = "-G " + cmd
 267         expand = True
 268     else:
 269         cmd = ["-G"] + cmd
 270         expand = False
 271
 272     cmd = p4_build_cmd(cmd)
 273     if verbose:
 274         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 275
 276     # Use a temporary file to avoid deadlocks without
 277     # subprocess.communicate(), which would put another copy
 278     # of stdout into memory.
 279     stdin_file = None
 280     if stdin is not None:
 281         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 282         if isinstance(stdin,basestring):
 283             stdin_file.write(stdin)
 284         else:
 285             for i in stdin:
 286                 stdin_file.write(i + '\n')
 287         stdin_file.flush()
 288         stdin_file.seek(0)
 289
 290     p4 = subprocess.Popen(cmd,
 291                           shell=expand,
 292                           stdin=stdin_file,
 293                           stdout=subprocess.PIPE)
 294
 295     result = []
 296     try:
 297         while True:
 298             entry = marshal.load(p4.stdout)
 299             if cb is not None:
 300                 cb(entry)
 301             else:
 302                 result.append(entry)
 303     except EOFError:
 304         pass
 305     exitCode = p4.wait()
 306     if exitCode != 0:
 307         entry = {}
 308         entry["p4ExitCode"] = exitCode
 309         result.append(entry)
 310
 311     return result
 312
 313 def p4Cmd(cmd):
 314     list = p4CmdList(cmd)
 315     result = {}
 316     for entry in list:
 317         result.update(entry)
 318     return result;
 319
 320 def p4Where(depotPath):
 321     if not depotPath.endswith("/"):
 322         depotPath += "/"
 323     depotPath = depotPath + "..."
 324     outputList = p4CmdList(["where", depotPath])
 325     output = None
 326     for entry in outputList:
 327         if "depotFile" in entry:
 328             if entry["depotFile"] == depotPath:
 329                 output = entry
 330                 break
 331         elif "data" in entry:
 332             data = entry.get("data")
 333             space = data.find(" ")
 334             if data[:space] == depotPath:
 335                 output = entry
 336                 break
 337     if output == None:
 338         return ""
 339     if output["code"] == "error":
 340         return ""
 341     clientPath = ""
 342     if "path" in output:
 343         clientPath = output.get("path")
 344     elif "data" in output:
 345         data = output.get("data")
 346         lastSpace = data.rfind(" ")
 347         clientPath = data[lastSpace + 1:]
 348
 349     if clientPath.endswith("..."):
 350         clientPath = clientPath[:-3]
 351     return clientPath
 352
 353 def currentGitBranch():
 354     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 355
 356 def isValidGitDir(path):
 357     if (os.path.exists(path + "/HEAD")
 358         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 359         return True;
 360     return False
 361
 362 def parseRevision(ref):
 363     return read_pipe("git rev-parse %s" % ref).strip()
 364
 365 def branchExists(ref):
 366     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 367                      ignore_error=True)
 368     return len(rev) > 0
 369
 370 def extractLogMessageFromGitCommit(commit):
 371     logMessage = ""
 372
 373     ## fixme: title is first line of commit, not 1st paragraph.
 374     foundTitle = False
 375     for log in read_pipe_lines("git cat-file commit %s" % commit):
 376        if not foundTitle:
 377            if len(log) == 1:
 378                foundTitle = True
 379            continue
 380
 381        logMessage += log
 382     return logMessage
 383
 384 def extractSettingsGitLog(log):
 385     values = {}
 386     for line in log.split("\n"):
 387         line = line.strip()
 388         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 389         if not m:
 390             continue
 391
 392         assignments = m.group(1).split (':')
 393         for a in assignments:
 394             vals = a.split ('=')
 395             key = vals[0].strip()
 396             val = ('='.join (vals[1:])).strip()
 397             if val.endswith ('\"') and val.startswith('"'):
 398                 val = val[1:-1]
 399
 400             values[key] = val
 401
 402     paths = values.get("depot-paths")
 403     if not paths:
 404         paths = values.get("depot-path")
 405     if paths:
 406         values['depot-paths'] = paths.split(',')
 407     return values
 408
 409 def gitBranchExists(branch):
 410     proc = subprocess.Popen(["git", "rev-parse", branch],
 411                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 412     return proc.wait() == 0;
 413
 414 _gitConfig = {}
 415 def gitConfig(key, args = None): # set args to "--bool", for instance
 416     if not _gitConfig.has_key(key):
 417         argsFilter = ""
 418         if args != None:
 419             argsFilter = "%s " % args
 420         cmd = "git config %s%s" % (argsFilter, key)
 421         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 422     return _gitConfig[key]
 423
 424 def gitConfigList(key):
 425     if not _gitConfig.has_key(key):
 426         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 427     return _gitConfig[key]
 428
 429 def p4BranchesInGit(branchesAreInRemotes = True):
 430     branches = {}
 431
 432     cmdline = "git rev-parse --symbolic "
 433     if branchesAreInRemotes:
 434         cmdline += " --remotes"
 435     else:
 436         cmdline += " --branches"
 437
 438     for line in read_pipe_lines(cmdline):
 439         line = line.strip()
 440
 441         ## only import to p4/
 442         if not line.startswith('p4/') or line == "p4/HEAD":
 443             continue
 444         branch = line
 445
 446         # strip off p4
 447         branch = re.sub ("^p4/", "", line)
 448
 449         branches[branch] = parseRevision(line)
 450     return branches
 451
 452 def findUpstreamBranchPoint(head = "HEAD"):
 453     branches = p4BranchesInGit()
 454     # map from depot-path to branch name
 455     branchByDepotPath = {}
 456     for branch in branches.keys():
 457         tip = branches[branch]
 458         log = extractLogMessageFromGitCommit(tip)
 459         settings = extractSettingsGitLog(log)
 460         if settings.has_key("depot-paths"):
 461             paths = ",".join(settings["depot-paths"])
 462             branchByDepotPath[paths] = "remotes/p4/" + branch
 463
 464     settings = None
 465     parent = 0
 466     while parent < 65535:
 467         commit = head + "~%s" % parent
 468         log = extractLogMessageFromGitCommit(commit)
 469         settings = extractSettingsGitLog(log)
 470         if settings.has_key("depot-paths"):
 471             paths = ",".join(settings["depot-paths"])
 472             if branchByDepotPath.has_key(paths):
 473                 return [branchByDepotPath[paths], settings]
 474
 475         parent = parent + 1
 476
 477     return ["", settings]
 478
 479 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 480     if not silent:
 481         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 482                % localRefPrefix)
 483
 484     originPrefix = "origin/p4/"
 485
 486     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 487         line = line.strip()
 488         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 489             continue
 490
 491         headName = line[len(originPrefix):]
 492         remoteHead = localRefPrefix + headName
 493         originHead = line
 494
 495         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 496         if (not original.has_key('depot-paths')
 497             or not original.has_key('change')):
 498             continue
 499
 500         update = False
 501         if not gitBranchExists(remoteHead):
 502             if verbose:
 503                 print "creating %s" % remoteHead
 504             update = True
 505         else:
 506             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 507             if settings.has_key('change') > 0:
 508                 if settings['depot-paths'] == original['depot-paths']:
 509                     originP4Change = int(original['change'])
 510                     p4Change = int(settings['change'])
 511                     if originP4Change > p4Change:
 512                         print ("%s (%s) is newer than %s (%s). "
 513                                "Updating p4 branch from origin."
 514                                % (originHead, originP4Change,
 515                                   remoteHead, p4Change))
 516                         update = True
 517                 else:
 518                     print ("Ignoring: %s was imported from %s while "
 519                            "%s was imported from %s"
 520                            % (originHead, ','.join(original['depot-paths']),
 521                               remoteHead, ','.join(settings['depot-paths'])))
 522
 523         if update:
 524             system("git update-ref %s %s" % (remoteHead, originHead))
 525
 526 def originP4BranchesExist():
 527         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 528
 529 def p4ChangesForPaths(depotPaths, changeRange):
 530     assert depotPaths
 531     cmd = ['changes']
 532     for p in depotPaths:
 533         cmd += ["%s...%s" % (p, changeRange)]
 534     output = p4_read_pipe_lines(cmd)
 535
 536     changes = {}
 537     for line in output:
 538         changeNum = int(line.split(" ")[1])
 539         changes[changeNum] = True
 540
 541     changelist = changes.keys()
 542     changelist.sort()
 543     return changelist
 544
 545 def p4PathStartsWith(path, prefix):
 546     # This method tries to remedy a potential mixed-case issue:
 547     #
 548     # If UserA adds  //depot/DirA/file1
 549     # and UserB adds //depot/dira/file2
 550     #
 551     # we may or may not have a problem. If you have core.ignorecase=true,
 552     # we treat DirA and dira as the same directory
 553     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 554     if ignorecase:
 555         return path.lower().startswith(prefix.lower())
 556     return path.startswith(prefix)
 557
 558 class Command:
 559     def __init__(self):
 560         self.usage = "usage: %prog [options]"
 561         self.needsGit = True
 562
 563 class P4UserMap:
 564     def __init__(self):
 565         self.userMapFromPerforceServer = False
 566
 567     def getUserCacheFilename(self):
 568         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 569         return home + "/.gitp4-usercache.txt"
 570
 571     def getUserMapFromPerforceServer(self):
 572         if self.userMapFromPerforceServer:
 573             return
 574         self.users = {}
 575         self.emails = {}
 576
 577         for output in p4CmdList("users"):
 578             if not output.has_key("User"):
 579                 continue
 580             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 581             self.emails[output["Email"]] = output["User"]
 582
 583
 584         s = ''
 585         for (key, val) in self.users.items():
 586             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 587
 588         open(self.getUserCacheFilename(), "wb").write(s)
 589         self.userMapFromPerforceServer = True
 590
 591     def loadUserMapFromCache(self):
 592         self.users = {}
 593         self.userMapFromPerforceServer = False
 594         try:
 595             cache = open(self.getUserCacheFilename(), "rb")
 596             lines = cache.readlines()
 597             cache.close()
 598             for line in lines:
 599                 entry = line.strip().split("\t")
 600                 self.users[entry[0]] = entry[1]
 601         except IOError:
 602             self.getUserMapFromPerforceServer()
 603
 604 class P4Debug(Command):
 605     def __init__(self):
 606         Command.__init__(self)
 607         self.options = [
 608             optparse.make_option("--verbose", dest="verbose", action="store_true",
 609                                  default=False),
 610             ]
 611         self.description = "A tool to debug the output of p4 -G."
 612         self.needsGit = False
 613         self.verbose = False
 614
 615     def run(self, args):
 616         j = 0
 617         for output in p4CmdList(args):
 618             print 'Element: %d' % j
 619             j += 1
 620             print output
 621         return True
 622
 623 class P4RollBack(Command):
 624     def __init__(self):
 625         Command.__init__(self)
 626         self.options = [
 627             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 628             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 629         ]
 630         self.description = "A tool to debug the multi-branch import. Don't use :)"
 631         self.verbose = False
 632         self.rollbackLocalBranches = False
 633
 634     def run(self, args):
 635         if len(args) != 1:
 636             return False
 637         maxChange = int(args[0])
 638
 639         if "p4ExitCode" in p4Cmd("changes -m 1"):
 640             die("Problems executing p4");
 641
 642         if self.rollbackLocalBranches:
 643             refPrefix = "refs/heads/"
 644             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 645         else:
 646             refPrefix = "refs/remotes/"
 647             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 648
 649         for line in lines:
 650             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 651                 line = line.strip()
 652                 ref = refPrefix + line
 653                 log = extractLogMessageFromGitCommit(ref)
 654                 settings = extractSettingsGitLog(log)
 655
 656                 depotPaths = settings['depot-paths']
 657                 change = settings['change']
 658
 659                 changed = False
 660
 661                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 662                                                            for p in depotPaths]))) == 0:
 663                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 664                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 665                     continue
 666
 667                 while change and int(change) > maxChange:
 668                     changed = True
 669                     if self.verbose:
 670                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 671                     system("git update-ref %s \"%s^\"" % (ref, ref))
 672                     log = extractLogMessageFromGitCommit(ref)
 673                     settings =  extractSettingsGitLog(log)
 674
 675
 676                     depotPaths = settings['depot-paths']
 677                     change = settings['change']
 678
 679                 if changed:
 680                     print "%s rewound to %s" % (ref, change)
 681
 682         return True
 683
 684 class P4Submit(Command, P4UserMap):
 685     def __init__(self):
 686         Command.__init__(self)
 687         P4UserMap.__init__(self)
 688         self.options = [
 689                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 690                 optparse.make_option("--origin", dest="origin"),
 691                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 692                 # preserve the user, requires relevant p4 permissions
 693                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 694         ]
 695         self.description = "Submit changes from git to the perforce depot."
 696         self.usage += " [name of git branch to submit into perforce depot]"
 697         self.interactive = True
 698         self.origin = ""
 699         self.detectRenames = False
 700         self.verbose = False
 701         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 702         self.isWindows = (platform.system() == "Windows")
 703         self.myP4UserId = None
 704
 705     def check(self):
 706         if len(p4CmdList("opened ...")) > 0:
 707             die("You have files opened with perforce! Close them before starting the sync.")
 708
 709     # replaces everything between 'Description:' and the next P4 submit template field with the
 710     # commit message
 711     def prepareLogMessage(self, template, message):
 712         result = ""
 713
 714         inDescriptionSection = False
 715
 716         for line in template.split("\n"):
 717             if line.startswith("#"):
 718                 result += line + "\n"
 719                 continue
 720
 721             if inDescriptionSection:
 722                 if line.startswith("Files:") or line.startswith("Jobs:"):
 723                     inDescriptionSection = False
 724                 else:
 725                     continue
 726             else:
 727                 if line.startswith("Description:"):
 728                     inDescriptionSection = True
 729                     line += "\n"
 730                     for messageLine in message.split("\n"):
 731                         line += "\t" + messageLine + "\n"
 732
 733             result += line + "\n"
 734
 735         return result
 736
 737     def p4UserForCommit(self,id):
 738         # Return the tuple (perforce user,git email) for a given git commit id
 739         self.getUserMapFromPerforceServer()
 740         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 741         gitEmail = gitEmail.strip()
 742         if not self.emails.has_key(gitEmail):
 743             return (None,gitEmail)
 744         else:
 745             return (self.emails[gitEmail],gitEmail)
 746
 747     def checkValidP4Users(self,commits):
 748         # check if any git authors cannot be mapped to p4 users
 749         for id in commits:
 750             (user,email) = self.p4UserForCommit(id)
 751             if not user:
 752                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 753                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 754                     print "%s" % msg
 755                 else:
 756                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 757
 758     def lastP4Changelist(self):
 759         # Get back the last changelist number submitted in this client spec. This
 760         # then gets used to patch up the username in the change. If the same
 761         # client spec is being used by multiple processes then this might go
 762         # wrong.
 763         results = p4CmdList("client -o")        # find the current client
 764         client = None
 765         for r in results:
 766             if r.has_key('Client'):
 767                 client = r['Client']
 768                 break
 769         if not client:
 770             die("could not get client spec")
 771         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 772         for r in results:
 773             if r.has_key('change'):
 774                 return r['change']
 775         die("Could not get changelist number for last submit - cannot patch up user details")
 776
 777     def modifyChangelistUser(self, changelist, newUser):
 778         # fixup the user field of a changelist after it has been submitted.
 779         changes = p4CmdList("change -o %s" % changelist)
 780         if len(changes) != 1:
 781             die("Bad output from p4 change modifying %s to user %s" %
 782                 (changelist, newUser))
 783
 784         c = changes[0]
 785         if c['User'] == newUser: return   # nothing to do
 786         c['User'] = newUser
 787         input = marshal.dumps(c)
 788
 789         result = p4CmdList("change -f -i", stdin=input)
 790         for r in result:
 791             if r.has_key('code'):
 792                 if r['code'] == 'error':
 793                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 794             if r.has_key('data'):
 795                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 796                 return
 797         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 798
 799     def canChangeChangelists(self):
 800         # check to see if we have p4 admin or super-user permissions, either of
 801         # which are required to modify changelists.
 802         results = p4CmdList(["protects", self.depotPath])
 803         for r in results:
 804             if r.has_key('perm'):
 805                 if r['perm'] == 'admin':
 806                     return 1
 807                 if r['perm'] == 'super':
 808                     return 1
 809         return 0
 810
 811     def p4UserId(self):
 812         if self.myP4UserId:
 813             return self.myP4UserId
 814
 815         results = p4CmdList("user -o")
 816         for r in results:
 817             if r.has_key('User'):
 818                 self.myP4UserId = r['User']
 819                 return r['User']
 820         die("Could not find your p4 user id")
 821
 822     def p4UserIsMe(self, p4User):
 823         # return True if the given p4 user is actually me
 824         me = self.p4UserId()
 825         if not p4User or p4User != me:
 826             return False
 827         else:
 828             return True
 829
 830     def prepareSubmitTemplate(self):
 831         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 832         template = ""
 833         inFilesSection = False
 834         for line in p4_read_pipe_lines(['change', '-o']):
 835             if line.endswith("\r\n"):
 836                 line = line[:-2] + "\n"
 837             if inFilesSection:
 838                 if line.startswith("\t"):
 839                     # path starts and ends with a tab
 840                     path = line[1:]
 841                     lastTab = path.rfind("\t")
 842                     if lastTab != -1:
 843                         path = path[:lastTab]
 844                         if not p4PathStartsWith(path, self.depotPath):
 845                             continue
 846                 else:
 847                     inFilesSection = False
 848             else:
 849                 if line.startswith("Files:"):
 850                     inFilesSection = True
 851
 852             template += line
 853
 854         return template
 855
 856     def edit_template(self, template_file):
 857         """Invoke the editor to let the user change the submission
 858            message.  Return true if okay to continue with the submit."""
 859
 860         # if configured to skip the editing part, just submit
 861         if gitConfig("git-p4.skipSubmitEdit") == "true":
 862             return True
 863
 864         # look at the modification time, to check later if the user saved
 865         # the file
 866         mtime = os.stat(template_file).st_mtime
 867
 868         # invoke the editor
 869         if os.environ.has_key("P4EDITOR"):
 870             editor = os.environ.get("P4EDITOR")
 871         else:
 872             editor = read_pipe("git var GIT_EDITOR").strip()
 873         system(editor + " " + template_file)
 874
 875         # If the file was not saved, prompt to see if this patch should
 876         # be skipped.  But skip this verification step if configured so.
 877         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 878             return True
 879
 880         # modification time updated means user saved the file
 881         if os.stat(template_file).st_mtime > mtime:
 882             return True
 883
 884         while True:
 885             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 886             if response == 'y':
 887                 return True
 888             if response == 'n':
 889                 return False
 890
 891     def applyCommit(self, id):
 892         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 893
 894         (p4User, gitEmail) = self.p4UserForCommit(id)
 895
 896         if not self.detectRenames:
 897             # If not explicitly set check the config variable
 898             self.detectRenames = gitConfig("git-p4.detectRenames")
 899
 900         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 901             diffOpts = ""
 902         elif self.detectRenames.lower() == "true":
 903             diffOpts = "-M"
 904         else:
 905             diffOpts = "-M%s" % self.detectRenames
 906
 907         detectCopies = gitConfig("git-p4.detectCopies")
 908         if detectCopies.lower() == "true":
 909             diffOpts += " -C"
 910         elif detectCopies != "" and detectCopies.lower() != "false":
 911             diffOpts += " -C%s" % detectCopies
 912
 913         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 914             diffOpts += " --find-copies-harder"
 915
 916         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 917         filesToAdd = set()
 918         filesToDelete = set()
 919         editedFiles = set()
 920         filesToChangeExecBit = {}
 921         for line in diff:
 922             diff = parseDiffTreeEntry(line)
 923             modifier = diff['status']
 924             path = diff['src']
 925             if modifier == "M":
 926                 p4_edit(path)
 927                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 928                     filesToChangeExecBit[path] = diff['dst_mode']
 929                 editedFiles.add(path)
 930             elif modifier == "A":
 931                 filesToAdd.add(path)
 932                 filesToChangeExecBit[path] = diff['dst_mode']
 933                 if path in filesToDelete:
 934                     filesToDelete.remove(path)
 935             elif modifier == "D":
 936                 filesToDelete.add(path)
 937                 if path in filesToAdd:
 938                     filesToAdd.remove(path)
 939             elif modifier == "C":
 940                 src, dest = diff['src'], diff['dst']
 941                 p4_integrate(src, dest)
 942                 if diff['src_sha1'] != diff['dst_sha1']:
 943                     p4_edit(dest)
 944                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 945                     p4_edit(dest)
 946                     filesToChangeExecBit[dest] = diff['dst_mode']
 947                 os.unlink(dest)
 948                 editedFiles.add(dest)
 949             elif modifier == "R":
 950                 src, dest = diff['src'], diff['dst']
 951                 p4_integrate(src, dest)
 952                 if diff['src_sha1'] != diff['dst_sha1']:
 953                     p4_edit(dest)
 954                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 955                     p4_edit(dest)
 956                     filesToChangeExecBit[dest] = diff['dst_mode']
 957                 os.unlink(dest)
 958                 editedFiles.add(dest)
 959                 filesToDelete.add(src)
 960             else:
 961                 die("unknown modifier %s for %s" % (modifier, path))
 962
 963         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 964         patchcmd = diffcmd + " | git apply "
 965         tryPatchCmd = patchcmd + "--check -"
 966         applyPatchCmd = patchcmd + "--check --apply -"
 967
 968         if os.system(tryPatchCmd) != 0:
 969             print "Unfortunately applying the change failed!"
 970             print "What do you want to do?"
 971             response = "x"
 972             while response != "s" and response != "a" and response != "w":
 973                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 974                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 975             if response == "s":
 976                 print "Skipping! Good luck with the next patches..."
 977                 for f in editedFiles:
 978                     p4_revert(f)
 979                 for f in filesToAdd:
 980                     os.remove(f)
 981                 return
 982             elif response == "a":
 983                 os.system(applyPatchCmd)
 984                 if len(filesToAdd) > 0:
 985                     print "You may also want to call p4 add on the following files:"
 986                     print " ".join(filesToAdd)
 987                 if len(filesToDelete):
 988                     print "The following files should be scheduled for deletion with p4 delete:"
 989                     print " ".join(filesToDelete)
 990                 die("Please resolve and submit the conflict manually and "
 991                     + "continue afterwards with git-p4 submit --continue")
 992             elif response == "w":
 993                 system(diffcmd + " > patch.txt")
 994                 print "Patch saved to patch.txt in %s !" % self.clientPath
 995                 die("Please resolve and submit the conflict manually and "
 996                     "continue afterwards with git-p4 submit --continue")
 997
 998         system(applyPatchCmd)
 999
1000         for f in filesToAdd:
1001             p4_add(f)
1002         for f in filesToDelete:
1003             p4_revert(f)
1004             p4_delete(f)
1005
1006         # Set/clear executable bits
1007         for f in filesToChangeExecBit.keys():
1008             mode = filesToChangeExecBit[f]
1009             setP4ExecBit(f, mode)
1010
1011         logMessage = extractLogMessageFromGitCommit(id)
1012         logMessage = logMessage.strip()
1013
1014         template = self.prepareSubmitTemplate()
1015
1016         if self.interactive:
1017             submitTemplate = self.prepareLogMessage(template, logMessage)
1018
1019             if self.preserveUser:
1020                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1021
1022             if os.environ.has_key("P4DIFF"):
1023                 del(os.environ["P4DIFF"])
1024             diff = ""
1025             for editedFile in editedFiles:
1026                 diff += p4_read_pipe(['diff', '-du', editedFile])
1027
1028             newdiff = ""
1029             for newFile in filesToAdd:
1030                 newdiff += "==== new file ====\n"
1031                 newdiff += "--- /dev/null\n"
1032                 newdiff += "+++ %s\n" % newFile
1033                 f = open(newFile, "r")
1034                 for line in f.readlines():
1035                     newdiff += "+" + line
1036                 f.close()
1037
1038             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1039                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1040                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1041                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1042
1043             separatorLine = "######## everything below this line is just the diff #######\n"
1044
1045             (handle, fileName) = tempfile.mkstemp()
1046             tmpFile = os.fdopen(handle, "w+")
1047             if self.isWindows:
1048                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1049                 separatorLine = separatorLine.replace("\n", "\r\n")
1050                 newdiff = newdiff.replace("\n", "\r\n")
1051             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1052             tmpFile.close()
1053
1054             if self.edit_template(fileName):
1055                 # read the edited message and submit
1056                 tmpFile = open(fileName, "rb")
1057                 message = tmpFile.read()
1058                 tmpFile.close()
1059                 submitTemplate = message[:message.index(separatorLine)]
1060                 if self.isWindows:
1061                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1062                 p4_write_pipe(['submit', '-i'], submitTemplate)
1063
1064                 if self.preserveUser:
1065                     if p4User:
1066                         # Get last changelist number. Cannot easily get it from
1067                         # the submit command output as the output is
1068                         # unmarshalled.
1069                         changelist = self.lastP4Changelist()
1070                         self.modifyChangelistUser(changelist, p4User)
1071             else:
1072                 # skip this patch
1073                 print "Submission cancelled, undoing p4 changes."
1074                 for f in editedFiles:
1075                     p4_revert(f)
1076                 for f in filesToAdd:
1077                     p4_revert(f)
1078                     os.remove(f)
1079
1080             os.remove(fileName)
1081         else:
1082             fileName = "submit.txt"
1083             file = open(fileName, "w+")
1084             file.write(self.prepareLogMessage(template, logMessage))
1085             file.close()
1086             print ("Perforce submit template written as %s. "
1087                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1088                    % (fileName, fileName))
1089
1090     def run(self, args):
1091         if len(args) == 0:
1092             self.master = currentGitBranch()
1093             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1094                 die("Detecting current git branch failed!")
1095         elif len(args) == 1:
1096             self.master = args[0]
1097             if not branchExists(self.master):
1098                 die("Branch %s does not exist" % self.master)
1099         else:
1100             return False
1101
1102         allowSubmit = gitConfig("git-p4.allowSubmit")
1103         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1104             die("%s is not in git-p4.allowSubmit" % self.master)
1105
1106         [upstream, settings] = findUpstreamBranchPoint()
1107         self.depotPath = settings['depot-paths'][0]
1108         if len(self.origin) == 0:
1109             self.origin = upstream
1110
1111         if self.preserveUser:
1112             if not self.canChangeChangelists():
1113                 die("Cannot preserve user names without p4 super-user or admin permissions")
1114
1115         if self.verbose:
1116             print "Origin branch is " + self.origin
1117
1118         if len(self.depotPath) == 0:
1119             print "Internal error: cannot locate perforce depot path from existing branches"
1120             sys.exit(128)
1121
1122         self.clientPath = p4Where(self.depotPath)
1123
1124         if len(self.clientPath) == 0:
1125             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1126             sys.exit(128)
1127
1128         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1129         self.oldWorkingDirectory = os.getcwd()
1130
1131         # ensure the clientPath exists
1132         if not os.path.exists(self.clientPath):
1133             os.makedirs(self.clientPath)
1134
1135         chdir(self.clientPath)
1136         print "Synchronizing p4 checkout..."
1137         p4_sync("...")
1138         self.check()
1139
1140         commits = []
1141         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1142             commits.append(line.strip())
1143         commits.reverse()
1144
1145         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1146             self.checkAuthorship = False
1147         else:
1148             self.checkAuthorship = True
1149
1150         if self.preserveUser:
1151             self.checkValidP4Users(commits)
1152
1153         while len(commits) > 0:
1154             commit = commits[0]
1155             commits = commits[1:]
1156             self.applyCommit(commit)
1157             if not self.interactive:
1158                 break
1159
1160         if len(commits) == 0:
1161             print "All changes applied!"
1162             chdir(self.oldWorkingDirectory)
1163
1164             sync = P4Sync()
1165             sync.run([])
1166
1167             rebase = P4Rebase()
1168             rebase.rebase()
1169
1170         return True
1171
1172 class View(object):
1173     """Represent a p4 view ("p4 help views"), and map files in a
1174        repo according to the view."""
1175
1176     class Path(object):
1177         """A depot or client path, possibly containing wildcards.
1178            The only one supported is ... at the end, currently.
1179            Initialize with the full path, with //depot or //client."""
1180
1181         def __init__(self, path, is_depot):
1182             self.path = path
1183             self.is_depot = is_depot
1184             self.find_wildcards()
1185             # remember the prefix bit, useful for relative mappings
1186             m = re.match("(//[^/]+/)", self.path)
1187             if not m:
1188                 die("Path %s does not start with //prefix/" % self.path)
1189             prefix = m.group(1)
1190             if not self.is_depot:
1191                 # strip //client/ on client paths
1192                 self.path = self.path[len(prefix):]
1193
1194         def find_wildcards(self):
1195             """Make sure wildcards are valid, and set up internal
1196                variables."""
1197
1198             self.ends_triple_dot = False
1199             # There are three wildcards allowed in p4 views
1200             # (see "p4 help views").  This code knows how to
1201             # handle "..." (only at the end), but cannot deal with
1202             # "%%n" or "*".  Only check the depot_side, as p4 should
1203             # validate that the client_side matches too.
1204             if re.search(r'%%[1-9]', self.path):
1205                 die("Can't handle %%n wildcards in view: %s" % self.path)
1206             if self.path.find("*") >= 0:
1207                 die("Can't handle * wildcards in view: %s" % self.path)
1208             triple_dot_index = self.path.find("...")
1209             if triple_dot_index >= 0:
1210                 if not self.path.endswith("..."):
1211                     die("Can handle ... wildcard only at end of path: %s" %
1212                         self.path)
1213                 self.ends_triple_dot = True
1214
1215         def ensure_compatible(self, other_path):
1216             """Make sure the wildcards agree."""
1217             if self.ends_triple_dot != other_path.ends_triple_dot:
1218                  die("Both paths must end with ... if either does;\n" +
1219                      "paths: %s %s" % (self.path, other_path.path))
1220
1221         def match_wildcards(self, test_path):
1222             """See if this test_path matches us, and fill in the value
1223                of the wildcards if so.  Returns a tuple of
1224                (True|False, wildcards[]).  For now, only the ... at end
1225                is supported, so at most one wildcard."""
1226             if self.ends_triple_dot:
1227                 dotless = self.path[:-3]
1228                 if test_path.startswith(dotless):
1229                     wildcard = test_path[len(dotless):]
1230                     return (True, [ wildcard ])
1231             else:
1232                 if test_path == self.path:
1233                     return (True, [])
1234             return (False, [])
1235
1236         def match(self, test_path):
1237             """Just return if it matches; don't bother with the wildcards."""
1238             b, _ = self.match_wildcards(test_path)
1239             return b
1240
1241         def fill_in_wildcards(self, wildcards):
1242             """Return the relative path, with the wildcards filled in
1243                if there are any."""
1244             if self.ends_triple_dot:
1245                 return self.path[:-3] + wildcards[0]
1246             else:
1247                 return self.path
1248
1249     class Mapping(object):
1250         def __init__(self, depot_side, client_side, overlay, exclude):
1251             # depot_side is without the trailing /... if it had one
1252             self.depot_side = View.Path(depot_side, is_depot=True)
1253             self.client_side = View.Path(client_side, is_depot=False)
1254             self.overlay = overlay  # started with "+"
1255             self.exclude = exclude  # started with "-"
1256             assert not (self.overlay and self.exclude)
1257             self.depot_side.ensure_compatible(self.client_side)
1258
1259         def __str__(self):
1260             c = " "
1261             if self.overlay:
1262                 c = "+"
1263             if self.exclude:
1264                 c = "-"
1265             return "View.Mapping: %s%s -> %s" % \
1266                    (c, self.depot_side, self.client_side)
1267
1268         def map_depot_to_client(self, depot_path):
1269             """Calculate the client path if using this mapping on the
1270                given depot path; does not consider the effect of other
1271                mappings in a view.  Even excluded mappings are returned."""
1272             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1273             if not matches:
1274                 return ""
1275             client_path = self.client_side.fill_in_wildcards(wildcards)
1276             return client_path
1277
1278     #
1279     # View methods
1280     #
1281     def __init__(self):
1282         self.mappings = []
1283
1284     def append(self, view_line):
1285         """Parse a view line, splitting it into depot and client
1286            sides.  Append to self.mappings, preserving order."""
1287
1288         # Split the view line into exactly two words.  P4 enforces
1289         # structure on these lines that simplifies this quite a bit.
1290         #
1291         # Either or both words may be double-quoted.
1292         # Single quotes do not matter.
1293         # Double-quote marks cannot occur inside the words.
1294         # A + or - prefix is also inside the quotes.
1295         # There are no quotes unless they contain a space.
1296         # The line is already white-space stripped.
1297         # The two words are separated by a single space.
1298         #
1299         if view_line[0] == '"':
1300             # First word is double quoted.  Find its end.
1301             close_quote_index = view_line.find('"', 1)
1302             if close_quote_index <= 0:
1303                 die("No first-word closing quote found: %s" % view_line)
1304             depot_side = view_line[1:close_quote_index]
1305             # skip closing quote and space
1306             rhs_index = close_quote_index + 1 + 1
1307         else:
1308             space_index = view_line.find(" ")
1309             if space_index <= 0:
1310                 die("No word-splitting space found: %s" % view_line)
1311             depot_side = view_line[0:space_index]
1312             rhs_index = space_index + 1
1313
1314         if view_line[rhs_index] == '"':
1315             # Second word is double quoted.  Make sure there is a
1316             # double quote at the end too.
1317             if not view_line.endswith('"'):
1318                 die("View line with rhs quote should end with one: %s" %
1319                     view_line)
1320             # skip the quotes
1321             client_side = view_line[rhs_index+1:-1]
1322         else:
1323             client_side = view_line[rhs_index:]
1324
1325         # prefix + means overlay on previous mapping
1326         overlay = False
1327         if depot_side.startswith("+"):
1328             overlay = True
1329             depot_side = depot_side[1:]
1330
1331         # prefix - means exclude this path
1332         exclude = False
1333         if depot_side.startswith("-"):
1334             exclude = True
1335             depot_side = depot_side[1:]
1336
1337         m = View.Mapping(depot_side, client_side, overlay, exclude)
1338         self.mappings.append(m)
1339
1340     def map_in_client(self, depot_path):
1341         """Return the relative location in the client where this
1342            depot file should live.  Returns "" if the file should
1343            not be mapped in the client."""
1344
1345         paths_filled = []
1346         client_path = ""
1347
1348         # look at later entries first
1349         for m in self.mappings[::-1]:
1350
1351             # see where will this path end up in the client
1352             p = m.map_depot_to_client(depot_path)
1353
1354             if p == "":
1355                 # Depot path does not belong in client.  Must remember
1356                 # this, as previous items should not cause files to
1357                 # exist in this path either.  Remember that the list is
1358                 # being walked from the end, which has higher precedence.
1359                 # Overlap mappings do not exclude previous mappings.
1360                 if not m.overlay:
1361                     paths_filled.append(m.client_side)
1362
1363             else:
1364                 # This mapping matched; no need to search any further.
1365                 # But, the mapping could be rejected if the client path
1366                 # has already been claimed by an earlier mapping.
1367                 already_mapped_in_client = False
1368                 for f in paths_filled:
1369                     # this is View.Path.match
1370                     if f.match(p):
1371                         already_mapped_in_client = True
1372                         break
1373                 if not already_mapped_in_client:
1374                     # Include this file, unless it is from a line that
1375                     # explicitly said to exclude it.
1376                     if not m.exclude:
1377                         client_path = p
1378
1379                 # a match, even if rejected, always stops the search
1380                 break
1381
1382         return client_path
1383
1384 class P4Sync(Command, P4UserMap):
1385     delete_actions = ( "delete", "move/delete", "purge" )
1386
1387     def __init__(self):
1388         Command.__init__(self)
1389         P4UserMap.__init__(self)
1390         self.options = [
1391                 optparse.make_option("--branch", dest="branch"),
1392                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1393                 optparse.make_option("--changesfile", dest="changesFile"),
1394                 optparse.make_option("--silent", dest="silent", action="store_true"),
1395                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1396                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1397                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1398                                      help="Import into refs/heads/ , not refs/remotes"),
1399                 optparse.make_option("--max-changes", dest="maxChanges"),
1400                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1401                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1402                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1403                                      help="Only sync files that are included in the Perforce Client Spec")
1404         ]
1405         self.description = """Imports from Perforce into a git repository.\n
1406     example:
1407     //depot/my/project/ -- to import the current head
1408     //depot/my/project/@all -- to import everything
1409     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1410
1411     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1412
1413         self.usage += " //depot/path[@revRange]"
1414         self.silent = False
1415         self.createdBranches = set()
1416         self.committedChanges = set()
1417         self.branch = ""
1418         self.detectBranches = False
1419         self.detectLabels = False
1420         self.changesFile = ""
1421         self.syncWithOrigin = True
1422         self.verbose = False
1423         self.importIntoRemotes = True
1424         self.maxChanges = ""
1425         self.isWindows = (platform.system() == "Windows")
1426         self.keepRepoPath = False
1427         self.depotPaths = None
1428         self.p4BranchesInGit = []
1429         self.cloneExclude = []
1430         self.useClientSpec = False
1431         self.clientSpecDirs = None
1432
1433         if gitConfig("git-p4.syncFromOrigin") == "false":
1434             self.syncWithOrigin = False
1435
1436     #
1437     # P4 wildcards are not allowed in filenames.  P4 complains
1438     # if you simply add them, but you can force it with "-f", in
1439     # which case it translates them into %xx encoding internally.
1440     # Search for and fix just these four characters.  Do % last so
1441     # that fixing it does not inadvertently create new %-escapes.
1442     #
1443     def wildcard_decode(self, path):
1444         # Cannot have * in a filename in windows; untested as to
1445         # what p4 would do in such a case.
1446         if not self.isWindows:
1447             path = path.replace("%2A", "*")
1448         path = path.replace("%23", "#") \
1449                    .replace("%40", "@") \
1450                    .replace("%25", "%")
1451         return path
1452
1453     def extractFilesFromCommit(self, commit):
1454         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1455                              for path in self.cloneExclude]
1456         files = []
1457         fnum = 0
1458         while commit.has_key("depotFile%s" % fnum):
1459             path =  commit["depotFile%s" % fnum]
1460
1461             if [p for p in self.cloneExclude
1462                 if p4PathStartsWith(path, p)]:
1463                 found = False
1464             else:
1465                 found = [p for p in self.depotPaths
1466                          if p4PathStartsWith(path, p)]
1467             if not found:
1468                 fnum = fnum + 1
1469                 continue
1470
1471             file = {}
1472             file["path"] = path
1473             file["rev"] = commit["rev%s" % fnum]
1474             file["action"] = commit["action%s" % fnum]
1475             file["type"] = commit["type%s" % fnum]
1476             files.append(file)
1477             fnum = fnum + 1
1478         return files
1479
1480     def stripRepoPath(self, path, prefixes):
1481         if self.useClientSpec:
1482             return self.clientSpecDirs.map_in_client(path)
1483
1484         if self.keepRepoPath:
1485             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1486
1487         for p in prefixes:
1488             if p4PathStartsWith(path, p):
1489                 path = path[len(p):]
1490
1491         return path
1492
1493     def splitFilesIntoBranches(self, commit):
1494         branches = {}
1495         fnum = 0
1496         while commit.has_key("depotFile%s" % fnum):
1497             path =  commit["depotFile%s" % fnum]
1498             found = [p for p in self.depotPaths
1499                      if p4PathStartsWith(path, p)]
1500             if not found:
1501                 fnum = fnum + 1
1502                 continue
1503
1504             file = {}
1505             file["path"] = path
1506             file["rev"] = commit["rev%s" % fnum]
1507             file["action"] = commit["action%s" % fnum]
1508             file["type"] = commit["type%s" % fnum]
1509             fnum = fnum + 1
1510
1511             relPath = self.stripRepoPath(path, self.depotPaths)
1512
1513             for branch in self.knownBranches.keys():
1514
1515                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1516                 if relPath.startswith(branch + "/"):
1517                     if branch not in branches:
1518                         branches[branch] = []
1519                     branches[branch].append(file)
1520                     break
1521
1522         return branches
1523
1524     # output one file from the P4 stream
1525     # - helper for streamP4Files
1526
1527     def streamOneP4File(self, file, contents):
1528         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1529         relPath = self.wildcard_decode(relPath)
1530         if verbose:
1531             sys.stderr.write("%s\n" % relPath)
1532
1533         (type_base, type_mods) = split_p4_type(file["type"])
1534
1535         git_mode = "100644"
1536         if "x" in type_mods:
1537             git_mode = "100755"
1538         if type_base == "symlink":
1539             git_mode = "120000"
1540             # p4 print on a symlink contains "target\n"; remove the newline
1541             data = ''.join(contents)
1542             contents = [data[:-1]]
1543
1544         if type_base == "utf16":
1545             # p4 delivers different text in the python output to -G
1546             # than it does when using "print -o", or normal p4 client
1547             # operations.  utf16 is converted to ascii or utf8, perhaps.
1548             # But ascii text saved as -t utf16 is completely mangled.
1549             # Invoke print -o to get the real contents.
1550             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1551             contents = [ text ]
1552
1553         if type_base == "apple":
1554             # Apple filetype files will be streamed as a concatenation of
1555             # its appledouble header and the contents.  This is useless
1556             # on both macs and non-macs.  If using "print -q -o xx", it
1557             # will create "xx" with the data, and "%xx" with the header.
1558             # This is also not very useful.
1559             #
1560             # Ideally, someday, this script can learn how to generate
1561             # appledouble files directly and import those to git, but
1562             # non-mac machines can never find a use for apple filetype.
1563             print "\nIgnoring apple filetype file %s" % file['depotFile']
1564             return
1565
1566         # Perhaps windows wants unicode, utf16 newlines translated too;
1567         # but this is not doing it.
1568         if self.isWindows and type_base == "text":
1569             mangled = []
1570             for data in contents:
1571                 data = data.replace("\r\n", "\n")
1572                 mangled.append(data)
1573             contents = mangled
1574
1575         # Note that we do not try to de-mangle keywords on utf16 files,
1576         # even though in theory somebody may want that.
1577         if type_base in ("text", "unicode", "binary"):
1578             if "ko" in type_mods:
1579                 text = ''.join(contents)
1580                 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1581                 contents = [ text ]
1582             elif "k" in type_mods:
1583                 text = ''.join(contents)
1584                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1585                 contents = [ text ]
1586
1587         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1588
1589         # total length...
1590         length = 0
1591         for d in contents:
1592             length = length + len(d)
1593
1594         self.gitStream.write("data %d\n" % length)
1595         for d in contents:
1596             self.gitStream.write(d)
1597         self.gitStream.write("\n")
1598
1599     def streamOneP4Deletion(self, file):
1600         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1601         if verbose:
1602             sys.stderr.write("delete %s\n" % relPath)
1603         self.gitStream.write("D %s\n" % relPath)
1604
1605     # handle another chunk of streaming data
1606     def streamP4FilesCb(self, marshalled):
1607
1608         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1609             # start of a new file - output the old one first
1610             self.streamOneP4File(self.stream_file, self.stream_contents)
1611             self.stream_file = {}
1612             self.stream_contents = []
1613             self.stream_have_file_info = False
1614
1615         # pick up the new file information... for the
1616         # 'data' field we need to append to our array
1617         for k in marshalled.keys():
1618             if k == 'data':
1619                 self.stream_contents.append(marshalled['data'])
1620             else:
1621                 self.stream_file[k] = marshalled[k]
1622
1623         self.stream_have_file_info = True
1624
1625     # Stream directly from "p4 files" into "git fast-import"
1626     def streamP4Files(self, files):
1627         filesForCommit = []
1628         filesToRead = []
1629         filesToDelete = []
1630
1631         for f in files:
1632             # if using a client spec, only add the files that have
1633             # a path in the client
1634             if self.clientSpecDirs:
1635                 if self.clientSpecDirs.map_in_client(f['path']) == "":
1636                     continue
1637
1638             filesForCommit.append(f)
1639             if f['action'] in self.delete_actions:
1640                 filesToDelete.append(f)
1641             else:
1642                 filesToRead.append(f)
1643
1644         # deleted files...
1645         for f in filesToDelete:
1646             self.streamOneP4Deletion(f)
1647
1648         if len(filesToRead) > 0:
1649             self.stream_file = {}
1650             self.stream_contents = []
1651             self.stream_have_file_info = False
1652
1653             # curry self argument
1654             def streamP4FilesCbSelf(entry):
1655                 self.streamP4FilesCb(entry)
1656
1657             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1658
1659             p4CmdList(["-x", "-", "print"],
1660                       stdin=fileArgs,
1661                       cb=streamP4FilesCbSelf)
1662
1663             # do the last chunk
1664             if self.stream_file.has_key('depotFile'):
1665                 self.streamOneP4File(self.stream_file, self.stream_contents)
1666
1667     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1668         epoch = details["time"]
1669         author = details["user"]
1670         self.branchPrefixes = branchPrefixes
1671
1672         if self.verbose:
1673             print "commit into %s" % branch
1674
1675         # start with reading files; if that fails, we should not
1676         # create a commit.
1677         new_files = []
1678         for f in files:
1679             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1680                 new_files.append (f)
1681             else:
1682                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1683
1684         self.gitStream.write("commit %s\n" % branch)
1685 #        gitStream.write("mark :%s\n" % details["change"])
1686         self.committedChanges.add(int(details["change"]))
1687         committer = ""
1688         if author not in self.users:
1689             self.getUserMapFromPerforceServer()
1690         if author in self.users:
1691             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1692         else:
1693             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1694
1695         self.gitStream.write("committer %s\n" % committer)
1696
1697         self.gitStream.write("data <<EOT\n")
1698         self.gitStream.write(details["desc"])
1699         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1700                              % (','.join (branchPrefixes), details["change"]))
1701         if len(details['options']) > 0:
1702             self.gitStream.write(": options = %s" % details['options'])
1703         self.gitStream.write("]\nEOT\n\n")
1704
1705         if len(parent) > 0:
1706             if self.verbose:
1707                 print "parent %s" % parent
1708             self.gitStream.write("from %s\n" % parent)
1709
1710         self.streamP4Files(new_files)
1711         self.gitStream.write("\n")
1712
1713         change = int(details["change"])
1714
1715         if self.labels.has_key(change):
1716             label = self.labels[change]
1717             labelDetails = label[0]
1718             labelRevisions = label[1]
1719             if self.verbose:
1720                 print "Change %s is labelled %s" % (change, labelDetails)
1721
1722             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1723                                                     for p in branchPrefixes])
1724
1725             if len(files) == len(labelRevisions):
1726
1727                 cleanedFiles = {}
1728                 for info in files:
1729                     if info["action"] in self.delete_actions:
1730                         continue
1731                     cleanedFiles[info["depotFile"]] = info["rev"]
1732
1733                 if cleanedFiles == labelRevisions:
1734                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1735                     self.gitStream.write("from %s\n" % branch)
1736
1737                     owner = labelDetails["Owner"]
1738                     tagger = ""
1739                     if author in self.users:
1740                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1741                     else:
1742                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1743                     self.gitStream.write("tagger %s\n" % tagger)
1744
1745                     description = labelDetails["Description"]
1746                     self.gitStream.write("data %d\n" % len(description))
1747                     self.gitStream.write(description)
1748                     self.gitStream.write("\n")
1749
1750                 else:
1751                     if not self.silent:
1752                         print ("Tag %s does not match with change %s: files do not match."
1753                                % (labelDetails["label"], change))
1754
1755             else:
1756                 if not self.silent:
1757                     print ("Tag %s does not match with change %s: file count is different."
1758                            % (labelDetails["label"], change))
1759
1760     def getLabels(self):
1761         self.labels = {}
1762
1763         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
1764         if len(l) > 0 and not self.silent:
1765             print "Finding files belonging to labels in %s" % `self.depotPaths`
1766
1767         for output in l:
1768             label = output["label"]
1769             revisions = {}
1770             newestChange = 0
1771             if self.verbose:
1772                 print "Querying files for label %s" % label
1773             for file in p4CmdList(["files"] +
1774                                       ["%s...@%s" % (p, label)
1775                                           for p in self.depotPaths]):
1776                 revisions[file["depotFile"]] = file["rev"]
1777                 change = int(file["change"])
1778                 if change > newestChange:
1779                     newestChange = change
1780
1781             self.labels[newestChange] = [output, revisions]
1782
1783         if self.verbose:
1784             print "Label changes: %s" % self.labels.keys()
1785
1786     def guessProjectName(self):
1787         for p in self.depotPaths:
1788             if p.endswith("/"):
1789                 p = p[:-1]
1790             p = p[p.strip().rfind("/") + 1:]
1791             if not p.endswith("/"):
1792                p += "/"
1793             return p
1794
1795     def getBranchMapping(self):
1796         lostAndFoundBranches = set()
1797
1798         user = gitConfig("git-p4.branchUser")
1799         if len(user) > 0:
1800             command = "branches -u %s" % user
1801         else:
1802             command = "branches"
1803
1804         for info in p4CmdList(command):
1805             details = p4Cmd(["branch", "-o", info["branch"]])
1806             viewIdx = 0
1807             while details.has_key("View%s" % viewIdx):
1808                 paths = details["View%s" % viewIdx].split(" ")
1809                 viewIdx = viewIdx + 1
1810                 # require standard //depot/foo/... //depot/bar/... mapping
1811                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1812                     continue
1813                 source = paths[0]
1814                 destination = paths[1]
1815                 ## HACK
1816                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1817                     source = source[len(self.depotPaths[0]):-4]
1818                     destination = destination[len(self.depotPaths[0]):-4]
1819
1820                     if destination in self.knownBranches:
1821                         if not self.silent:
1822                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1823                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1824                         continue
1825
1826                     self.knownBranches[destination] = source
1827
1828                     lostAndFoundBranches.discard(destination)
1829
1830                     if source not in self.knownBranches:
1831                         lostAndFoundBranches.add(source)
1832
1833         # Perforce does not strictly require branches to be defined, so we also
1834         # check git config for a branch list.
1835         #
1836         # Example of branch definition in git config file:
1837         # [git-p4]
1838         #   branchList=main:branchA
1839         #   branchList=main:branchB
1840         #   branchList=branchA:branchC
1841         configBranches = gitConfigList("git-p4.branchList")
1842         for branch in configBranches:
1843             if branch:
1844                 (source, destination) = branch.split(":")
1845                 self.knownBranches[destination] = source
1846
1847                 lostAndFoundBranches.discard(destination)
1848
1849                 if source not in self.knownBranches:
1850                     lostAndFoundBranches.add(source)
1851
1852
1853         for branch in lostAndFoundBranches:
1854             self.knownBranches[branch] = branch
1855
1856     def getBranchMappingFromGitBranches(self):
1857         branches = p4BranchesInGit(self.importIntoRemotes)
1858         for branch in branches.keys():
1859             if branch == "master":
1860                 branch = "main"
1861             else:
1862                 branch = branch[len(self.projectName):]
1863             self.knownBranches[branch] = branch
1864
1865     def listExistingP4GitBranches(self):
1866         # branches holds mapping from name to commit
1867         branches = p4BranchesInGit(self.importIntoRemotes)
1868         self.p4BranchesInGit = branches.keys()
1869         for branch in branches.keys():
1870             self.initialParents[self.refPrefix + branch] = branches[branch]
1871
1872     def updateOptionDict(self, d):
1873         option_keys = {}
1874         if self.keepRepoPath:
1875             option_keys['keepRepoPath'] = 1
1876
1877         d["options"] = ' '.join(sorted(option_keys.keys()))
1878
1879     def readOptions(self, d):
1880         self.keepRepoPath = (d.has_key('options')
1881                              and ('keepRepoPath' in d['options']))
1882
1883     def gitRefForBranch(self, branch):
1884         if branch == "main":
1885             return self.refPrefix + "master"
1886
1887         if len(branch) <= 0:
1888             return branch
1889
1890         return self.refPrefix + self.projectName + branch
1891
1892     def gitCommitByP4Change(self, ref, change):
1893         if self.verbose:
1894             print "looking in ref " + ref + " for change %s using bisect..." % change
1895
1896         earliestCommit = ""
1897         latestCommit = parseRevision(ref)
1898
1899         while True:
1900             if self.verbose:
1901                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1902             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1903             if len(next) == 0:
1904                 if self.verbose:
1905                     print "argh"
1906                 return ""
1907             log = extractLogMessageFromGitCommit(next)
1908             settings = extractSettingsGitLog(log)
1909             currentChange = int(settings['change'])
1910             if self.verbose:
1911                 print "current change %s" % currentChange
1912
1913             if currentChange == change:
1914                 if self.verbose:
1915                     print "found %s" % next
1916                 return next
1917
1918             if currentChange < change:
1919                 earliestCommit = "^%s" % next
1920             else:
1921                 latestCommit = "%s" % next
1922
1923         return ""
1924
1925     def importNewBranch(self, branch, maxChange):
1926         # make fast-import flush all changes to disk and update the refs using the checkpoint
1927         # command so that we can try to find the branch parent in the git history
1928         self.gitStream.write("checkpoint\n\n");
1929         self.gitStream.flush();
1930         branchPrefix = self.depotPaths[0] + branch + "/"
1931         range = "@1,%s" % maxChange
1932         #print "prefix" + branchPrefix
1933         changes = p4ChangesForPaths([branchPrefix], range)
1934         if len(changes) <= 0:
1935             return False
1936         firstChange = changes[0]
1937         #print "first change in branch: %s" % firstChange
1938         sourceBranch = self.knownBranches[branch]
1939         sourceDepotPath = self.depotPaths[0] + sourceBranch
1940         sourceRef = self.gitRefForBranch(sourceBranch)
1941         #print "source " + sourceBranch
1942
1943         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1944         #print "branch parent: %s" % branchParentChange
1945         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1946         if len(gitParent) > 0:
1947             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1948             #print "parent git commit: %s" % gitParent
1949
1950         self.importChanges(changes)
1951         return True
1952
1953     def importChanges(self, changes):
1954         cnt = 1
1955         for change in changes:
1956             description = p4Cmd("describe %s" % change)
1957             self.updateOptionDict(description)
1958
1959             if not self.silent:
1960                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1961                 sys.stdout.flush()
1962             cnt = cnt + 1
1963
1964             try:
1965                 if self.detectBranches:
1966                     branches = self.splitFilesIntoBranches(description)
1967                     for branch in branches.keys():
1968                         ## HACK  --hwn
1969                         branchPrefix = self.depotPaths[0] + branch + "/"
1970
1971                         parent = ""
1972
1973                         filesForCommit = branches[branch]
1974
1975                         if self.verbose:
1976                             print "branch is %s" % branch
1977
1978                         self.updatedBranches.add(branch)
1979
1980                         if branch not in self.createdBranches:
1981                             self.createdBranches.add(branch)
1982                             parent = self.knownBranches[branch]
1983                             if parent == branch:
1984                                 parent = ""
1985                             else:
1986                                 fullBranch = self.projectName + branch
1987                                 if fullBranch not in self.p4BranchesInGit:
1988                                     if not self.silent:
1989                                         print("\n    Importing new branch %s" % fullBranch);
1990                                     if self.importNewBranch(branch, change - 1):
1991                                         parent = ""
1992                                         self.p4BranchesInGit.append(fullBranch)
1993                                     if not self.silent:
1994                                         print("\n    Resuming with change %s" % change);
1995
1996                                 if self.verbose:
1997                                     print "parent determined through known branches: %s" % parent
1998
1999                         branch = self.gitRefForBranch(branch)
2000                         parent = self.gitRefForBranch(parent)
2001
2002                         if self.verbose:
2003                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2004
2005                         if len(parent) == 0 and branch in self.initialParents:
2006                             parent = self.initialParents[branch]
2007                             del self.initialParents[branch]
2008
2009                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2010                 else:
2011                     files = self.extractFilesFromCommit(description)
2012                     self.commit(description, files, self.branch, self.depotPaths,
2013                                 self.initialParent)
2014                     self.initialParent = ""
2015             except IOError:
2016                 print self.gitError.read()
2017                 sys.exit(1)
2018
2019     def importHeadRevision(self, revision):
2020         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2021
2022         details = {}
2023         details["user"] = "git perforce import user"
2024         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2025                            % (' '.join(self.depotPaths), revision))
2026         details["change"] = revision
2027         newestRevision = 0
2028
2029         fileCnt = 0
2030         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2031
2032         for info in p4CmdList(["files"] + fileArgs):
2033
2034             if 'code' in info and info['code'] == 'error':
2035                 sys.stderr.write("p4 returned an error: %s\n"
2036                                  % info['data'])
2037                 if info['data'].find("must refer to client") >= 0:
2038                     sys.stderr.write("This particular p4 error is misleading.\n")
2039                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2040                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2041                 sys.exit(1)
2042             if 'p4ExitCode' in info:
2043                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2044                 sys.exit(1)
2045
2046
2047             change = int(info["change"])
2048             if change > newestRevision:
2049                 newestRevision = change
2050
2051             if info["action"] in self.delete_actions:
2052                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2053                 #fileCnt = fileCnt + 1
2054                 continue
2055
2056             for prop in ["depotFile", "rev", "action", "type" ]:
2057                 details["%s%s" % (prop, fileCnt)] = info[prop]
2058
2059             fileCnt = fileCnt + 1
2060
2061         details["change"] = newestRevision
2062
2063         # Use time from top-most change so that all git-p4 clones of
2064         # the same p4 repo have the same commit SHA1s.
2065         res = p4CmdList("describe -s %d" % newestRevision)
2066         newestTime = None
2067         for r in res:
2068             if r.has_key('time'):
2069                 newestTime = int(r['time'])
2070         if newestTime is None:
2071             die("\"describe -s\" on newest change %d did not give a time")
2072         details["time"] = newestTime
2073
2074         self.updateOptionDict(details)
2075         try:
2076             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2077         except IOError:
2078             print "IO error with git fast-import. Is your git version recent enough?"
2079             print self.gitError.read()
2080
2081
2082     def getClientSpec(self):
2083         specList = p4CmdList("client -o")
2084         if len(specList) != 1:
2085             die('Output from "client -o" is %d lines, expecting 1' %
2086                 len(specList))
2087
2088         # dictionary of all client parameters
2089         entry = specList[0]
2090
2091         # just the keys that start with "View"
2092         view_keys = [ k for k in entry.keys() if k.startswith("View") ]
2093
2094         # hold this new View
2095         view = View()
2096
2097         # append the lines, in order, to the view
2098         for view_num in range(len(view_keys)):
2099             k = "View%d" % view_num
2100             if k not in view_keys:
2101                 die("Expected view key %s missing" % k)
2102             view.append(entry[k])
2103
2104         self.clientSpecDirs = view
2105         if self.verbose:
2106             for i, m in enumerate(self.clientSpecDirs.mappings):
2107                     print "clientSpecDirs %d: %s" % (i, str(m))
2108
2109     def run(self, args):
2110         self.depotPaths = []
2111         self.changeRange = ""
2112         self.initialParent = ""
2113         self.previousDepotPaths = []
2114
2115         # map from branch depot path to parent branch
2116         self.knownBranches = {}
2117         self.initialParents = {}
2118         self.hasOrigin = originP4BranchesExist()
2119         if not self.syncWithOrigin:
2120             self.hasOrigin = False
2121
2122         if self.importIntoRemotes:
2123             self.refPrefix = "refs/remotes/p4/"
2124         else:
2125             self.refPrefix = "refs/heads/p4/"
2126
2127         if self.syncWithOrigin and self.hasOrigin:
2128             if not self.silent:
2129                 print "Syncing with origin first by calling git fetch origin"
2130             system("git fetch origin")
2131
2132         if len(self.branch) == 0:
2133             self.branch = self.refPrefix + "master"
2134             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2135                 system("git update-ref %s refs/heads/p4" % self.branch)
2136                 system("git branch -D p4");
2137             # create it /after/ importing, when master exists
2138             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2139                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2140
2141         if not self.useClientSpec:
2142             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2143                 self.useClientSpec = True
2144         if self.useClientSpec:
2145             self.getClientSpec()
2146
2147         # TODO: should always look at previous commits,
2148         # merge with previous imports, if possible.
2149         if args == []:
2150             if self.hasOrigin:
2151                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2152             self.listExistingP4GitBranches()
2153
2154             if len(self.p4BranchesInGit) > 1:
2155                 if not self.silent:
2156                     print "Importing from/into multiple branches"
2157                 self.detectBranches = True
2158
2159             if self.verbose:
2160                 print "branches: %s" % self.p4BranchesInGit
2161
2162             p4Change = 0
2163             for branch in self.p4BranchesInGit:
2164                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2165
2166                 settings = extractSettingsGitLog(logMsg)
2167
2168                 self.readOptions(settings)
2169                 if (settings.has_key('depot-paths')
2170                     and settings.has_key ('change')):
2171                     change = int(settings['change']) + 1
2172                     p4Change = max(p4Change, change)
2173
2174                     depotPaths = sorted(settings['depot-paths'])
2175                     if self.previousDepotPaths == []:
2176                         self.previousDepotPaths = depotPaths
2177                     else:
2178                         paths = []
2179                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2180                             prev_list = prev.split("/")
2181                             cur_list = cur.split("/")
2182                             for i in range(0, min(len(cur_list), len(prev_list))):
2183                                 if cur_list[i] <> prev_list[i]:
2184                                     i = i - 1
2185                                     break
2186
2187                             paths.append ("/".join(cur_list[:i + 1]))
2188
2189                         self.previousDepotPaths = paths
2190
2191             if p4Change > 0:
2192                 self.depotPaths = sorted(self.previousDepotPaths)
2193                 self.changeRange = "@%s,#head" % p4Change
2194                 if not self.detectBranches:
2195                     self.initialParent = parseRevision(self.branch)
2196                 if not self.silent and not self.detectBranches:
2197                     print "Performing incremental import into %s git branch" % self.branch
2198
2199         if not self.branch.startswith("refs/"):
2200             self.branch = "refs/heads/" + self.branch
2201
2202         if len(args) == 0 and self.depotPaths:
2203             if not self.silent:
2204                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2205         else:
2206             if self.depotPaths and self.depotPaths != args:
2207                 print ("previous import used depot path %s and now %s was specified. "
2208                        "This doesn't work!" % (' '.join (self.depotPaths),
2209                                                ' '.join (args)))
2210                 sys.exit(1)
2211
2212             self.depotPaths = sorted(args)
2213
2214         revision = ""
2215         self.users = {}
2216
2217         # Make sure no revision specifiers are used when --changesfile
2218         # is specified.
2219         bad_changesfile = False
2220         if len(self.changesFile) > 0:
2221             for p in self.depotPaths:
2222                 if p.find("@") >= 0 or p.find("#") >= 0:
2223                     bad_changesfile = True
2224                     break
2225         if bad_changesfile:
2226             die("Option --changesfile is incompatible with revision specifiers")
2227
2228         newPaths = []
2229         for p in self.depotPaths:
2230             if p.find("@") != -1:
2231                 atIdx = p.index("@")
2232                 self.changeRange = p[atIdx:]
2233                 if self.changeRange == "@all":
2234                     self.changeRange = ""
2235                 elif ',' not in self.changeRange:
2236                     revision = self.changeRange
2237                     self.changeRange = ""
2238                 p = p[:atIdx]
2239             elif p.find("#") != -1:
2240                 hashIdx = p.index("#")
2241                 revision = p[hashIdx:]
2242                 p = p[:hashIdx]
2243             elif self.previousDepotPaths == []:
2244                 # pay attention to changesfile, if given, else import
2245                 # the entire p4 tree at the head revision
2246                 if len(self.changesFile) == 0:
2247                     revision = "#head"
2248
2249             p = re.sub ("\.\.\.$", "", p)
2250             if not p.endswith("/"):
2251                 p += "/"
2252
2253             newPaths.append(p)
2254
2255         self.depotPaths = newPaths
2256
2257
2258         self.loadUserMapFromCache()
2259         self.labels = {}
2260         if self.detectLabels:
2261             self.getLabels();
2262
2263         if self.detectBranches:
2264             ## FIXME - what's a P4 projectName ?
2265             self.projectName = self.guessProjectName()
2266
2267             if self.hasOrigin:
2268                 self.getBranchMappingFromGitBranches()
2269             else:
2270                 self.getBranchMapping()
2271             if self.verbose:
2272                 print "p4-git branches: %s" % self.p4BranchesInGit
2273                 print "initial parents: %s" % self.initialParents
2274             for b in self.p4BranchesInGit:
2275                 if b != "master":
2276
2277                     ## FIXME
2278                     b = b[len(self.projectName):]
2279                 self.createdBranches.add(b)
2280
2281         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2282
2283         importProcess = subprocess.Popen(["git", "fast-import"],
2284                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2285                                          stderr=subprocess.PIPE);
2286         self.gitOutput = importProcess.stdout
2287         self.gitStream = importProcess.stdin
2288         self.gitError = importProcess.stderr
2289
2290         if revision:
2291             self.importHeadRevision(revision)
2292         else:
2293             changes = []
2294
2295             if len(self.changesFile) > 0:
2296                 output = open(self.changesFile).readlines()
2297                 changeSet = set()
2298                 for line in output:
2299                     changeSet.add(int(line))
2300
2301                 for change in changeSet:
2302                     changes.append(change)
2303
2304                 changes.sort()
2305             else:
2306                 # catch "git-p4 sync" with no new branches, in a repo that
2307                 # does not have any existing git-p4 branches
2308                 if len(args) == 0 and not self.p4BranchesInGit:
2309                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2310                 if self.verbose:
2311                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2312                                                               self.changeRange)
2313                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2314
2315                 if len(self.maxChanges) > 0:
2316                     changes = changes[:min(int(self.maxChanges), len(changes))]
2317
2318             if len(changes) == 0:
2319                 if not self.silent:
2320                     print "No changes to import!"
2321                 return True
2322
2323             if not self.silent and not self.detectBranches:
2324                 print "Import destination: %s" % self.branch
2325
2326             self.updatedBranches = set()
2327
2328             self.importChanges(changes)
2329
2330             if not self.silent:
2331                 print ""
2332                 if len(self.updatedBranches) > 0:
2333                     sys.stdout.write("Updated branches: ")
2334                     for b in self.updatedBranches:
2335                         sys.stdout.write("%s " % b)
2336                     sys.stdout.write("\n")
2337
2338         self.gitStream.close()
2339         if importProcess.wait() != 0:
2340             die("fast-import failed: %s" % self.gitError.read())
2341         self.gitOutput.close()
2342         self.gitError.close()
2343
2344         return True
2345
2346 class P4Rebase(Command):
2347     def __init__(self):
2348         Command.__init__(self)
2349         self.options = [ ]
2350         self.description = ("Fetches the latest revision from perforce and "
2351                             + "rebases the current work (branch) against it")
2352         self.verbose = False
2353
2354     def run(self, args):
2355         sync = P4Sync()
2356         sync.run([])
2357
2358         return self.rebase()
2359
2360     def rebase(self):
2361         if os.system("git update-index --refresh") != 0:
2362             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2363         if len(read_pipe("git diff-index HEAD --")) > 0:
2364             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2365
2366         [upstream, settings] = findUpstreamBranchPoint()
2367         if len(upstream) == 0:
2368             die("Cannot find upstream branchpoint for rebase")
2369
2370         # the branchpoint may be p4/foo~3, so strip off the parent
2371         upstream = re.sub("~[0-9]+$", "", upstream)
2372
2373         print "Rebasing the current branch onto %s" % upstream
2374         oldHead = read_pipe("git rev-parse HEAD").strip()
2375         system("git rebase %s" % upstream)
2376         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2377         return True
2378
2379 class P4Clone(P4Sync):
2380     def __init__(self):
2381         P4Sync.__init__(self)
2382         self.description = "Creates a new git repository and imports from Perforce into it"
2383         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2384         self.options += [
2385             optparse.make_option("--destination", dest="cloneDestination",
2386                                  action='store', default=None,
2387                                  help="where to leave result of the clone"),
2388             optparse.make_option("-/", dest="cloneExclude",
2389                                  action="append", type="string",
2390                                  help="exclude depot path"),
2391             optparse.make_option("--bare", dest="cloneBare",
2392                                  action="store_true", default=False),
2393         ]
2394         self.cloneDestination = None
2395         self.needsGit = False
2396         self.cloneBare = False
2397
2398     # This is required for the "append" cloneExclude action
2399     def ensure_value(self, attr, value):
2400         if not hasattr(self, attr) or getattr(self, attr) is None:
2401             setattr(self, attr, value)
2402         return getattr(self, attr)
2403
2404     def defaultDestination(self, args):
2405         ## TODO: use common prefix of args?
2406         depotPath = args[0]
2407         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2408         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2409         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2410         depotDir = re.sub(r"/$", "", depotDir)
2411         return os.path.split(depotDir)[1]
2412
2413     def run(self, args):
2414         if len(args) < 1:
2415             return False
2416
2417         if self.keepRepoPath and not self.cloneDestination:
2418             sys.stderr.write("Must specify destination for --keep-path\n")
2419             sys.exit(1)
2420
2421         depotPaths = args
2422
2423         if not self.cloneDestination and len(depotPaths) > 1:
2424             self.cloneDestination = depotPaths[-1]
2425             depotPaths = depotPaths[:-1]
2426
2427         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2428         for p in depotPaths:
2429             if not p.startswith("//"):
2430                 return False
2431
2432         if not self.cloneDestination:
2433             self.cloneDestination = self.defaultDestination(args)
2434
2435         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2436
2437         if not os.path.exists(self.cloneDestination):
2438             os.makedirs(self.cloneDestination)
2439         chdir(self.cloneDestination)
2440
2441         init_cmd = [ "git", "init" ]
2442         if self.cloneBare:
2443             init_cmd.append("--bare")
2444         subprocess.check_call(init_cmd)
2445
2446         if not P4Sync.run(self, depotPaths):
2447             return False
2448         if self.branch != "master":
2449             if self.importIntoRemotes:
2450                 masterbranch = "refs/remotes/p4/master"
2451             else:
2452                 masterbranch = "refs/heads/p4/master"
2453             if gitBranchExists(masterbranch):
2454                 system("git branch master %s" % masterbranch)
2455                 if not self.cloneBare:
2456                     system("git checkout -f")
2457             else:
2458                 print "Could not detect main branch. No checkout/master branch created."
2459
2460         return True
2461
2462 class P4Branches(Command):
2463     def __init__(self):
2464         Command.__init__(self)
2465         self.options = [ ]
2466         self.description = ("Shows the git branches that hold imports and their "
2467                             + "corresponding perforce depot paths")
2468         self.verbose = False
2469
2470     def run(self, args):
2471         if originP4BranchesExist():
2472             createOrUpdateBranchesFromOrigin()
2473
2474         cmdline = "git rev-parse --symbolic "
2475         cmdline += " --remotes"
2476
2477         for line in read_pipe_lines(cmdline):
2478             line = line.strip()
2479
2480             if not line.startswith('p4/') or line == "p4/HEAD":
2481                 continue
2482             branch = line
2483
2484             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2485             settings = extractSettingsGitLog(log)
2486
2487             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2488         return True
2489
2490 class HelpFormatter(optparse.IndentedHelpFormatter):
2491     def __init__(self):
2492         optparse.IndentedHelpFormatter.__init__(self)
2493
2494     def format_description(self, description):
2495         if description:
2496             return description + "\n"
2497         else:
2498             return ""
2499
2500 def printUsage(commands):
2501     print "usage: %s <command> [options]" % sys.argv[0]
2502     print ""
2503     print "valid commands: %s" % ", ".join(commands)
2504     print ""
2505     print "Try %s <command> --help for command specific help." % sys.argv[0]
2506     print ""
2507
2508 commands = {
2509     "debug" : P4Debug,
2510     "submit" : P4Submit,
2511     "commit" : P4Submit,
2512     "sync" : P4Sync,
2513     "rebase" : P4Rebase,
2514     "clone" : P4Clone,
2515     "rollback" : P4RollBack,
2516     "branches" : P4Branches
2517 }
2518
2519
2520 def main():
2521     if len(sys.argv[1:]) == 0:
2522         printUsage(commands.keys())
2523         sys.exit(2)
2524
2525     cmd = ""
2526     cmdName = sys.argv[1]
2527     try:
2528         klass = commands[cmdName]
2529         cmd = klass()
2530     except KeyError:
2531         print "unknown command %s" % cmdName
2532         print ""
2533         printUsage(commands.keys())
2534         sys.exit(2)
2535
2536     options = cmd.options
2537     cmd.gitdir = os.environ.get("GIT_DIR", None)
2538
2539     args = sys.argv[2:]
2540
2541     if len(options) > 0:
2542         if cmd.needsGit:
2543             options.append(optparse.make_option("--git-dir", dest="gitdir"))
2544
2545         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2546                                        options,
2547                                        description = cmd.description,
2548                                        formatter = HelpFormatter())
2549
2550         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2551     global verbose
2552     verbose = cmd.verbose
2553     if cmd.needsGit:
2554         if cmd.gitdir == None:
2555             cmd.gitdir = os.path.abspath(".git")
2556             if not isValidGitDir(cmd.gitdir):
2557                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2558                 if os.path.exists(cmd.gitdir):
2559                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2560                     if len(cdup) > 0:
2561                         chdir(cdup);
2562
2563         if not isValidGitDir(cmd.gitdir):
2564             if isValidGitDir(cmd.gitdir + "/.git"):
2565                 cmd.gitdir += "/.git"
2566             else:
2567                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2568
2569         os.environ["GIT_DIR"] = cmd.gitdir
2570
2571     if not cmd.run(args):
2572         parser.print_help()
2573         sys.exit(2)
2574
2575
2576 if __name__ == '__main__':
2577     main()