contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s" % file)
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                 optparse.make_option("--origin", dest="origin"),
 469                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 470         ]
 471         self.description = "Submit changes from git to the perforce depot."
 472         self.usage += " [name of git branch to submit into perforce depot]"
 473         self.interactive = True
 474         self.origin = ""
 475         self.detectRename = False
 476         self.verbose = False
 477         self.isWindows = (platform.system() == "Windows")
 478
 479     def check(self):
 480         if len(p4CmdList("opened ...")) > 0:
 481             die("You have files opened with perforce! Close them before starting the sync.")
 482
 483     # replaces everything between 'Description:' and the next P4 submit template field with the
 484     # commit message
 485     def prepareLogMessage(self, template, message):
 486         result = ""
 487
 488         inDescriptionSection = False
 489
 490         for line in template.split("\n"):
 491             if line.startswith("#"):
 492                 result += line + "\n"
 493                 continue
 494
 495             if inDescriptionSection:
 496                 if line.startswith("Files:"):
 497                     inDescriptionSection = False
 498                 else:
 499                     continue
 500             else:
 501                 if line.startswith("Description:"):
 502                     inDescriptionSection = True
 503                     line += "\n"
 504                     for messageLine in message.split("\n"):
 505                         line += "\t" + messageLine + "\n"
 506
 507             result += line + "\n"
 508
 509         return result
 510
 511     def prepareSubmitTemplate(self):
 512         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513         template = ""
 514         inFilesSection = False
 515         for line in read_pipe_lines("p4 change -o"):
 516             if inFilesSection:
 517                 if line.startswith("\t"):
 518                     # path starts and ends with a tab
 519                     path = line[1:]
 520                     lastTab = path.rfind("\t")
 521                     if lastTab != -1:
 522                         path = path[:lastTab]
 523                         if not path.startswith(self.depotPath):
 524                             continue
 525                 else:
 526                     inFilesSection = False
 527             else:
 528                 if line.startswith("Files:"):
 529                     inFilesSection = True
 530
 531             template += line
 532
 533         return template
 534
 535     def applyCommit(self, id):
 536         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 537         diffOpts = ("", "-M")[self.detectRename]
 538         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 539         filesToAdd = set()
 540         filesToDelete = set()
 541         editedFiles = set()
 542         filesToChangeExecBit = {}
 543         for line in diff:
 544             diff = parseDiffTreeEntry(line)
 545             modifier = diff['status']
 546             path = diff['src']
 547             if modifier == "M":
 548                 system("p4 edit \"%s\"" % path)
 549                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 550                     filesToChangeExecBit[path] = diff['dst_mode']
 551                 editedFiles.add(path)
 552             elif modifier == "A":
 553                 filesToAdd.add(path)
 554                 filesToChangeExecBit[path] = diff['dst_mode']
 555                 if path in filesToDelete:
 556                     filesToDelete.remove(path)
 557             elif modifier == "D":
 558                 filesToDelete.add(path)
 559                 if path in filesToAdd:
 560                     filesToAdd.remove(path)
 561             elif modifier == "R":
 562                 src, dest = diff['src'], diff['dst']
 563                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 564                 system("p4 edit \"%s\"" % (dest))
 565                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 566                     filesToChangeExecBit[dest] = diff['dst_mode']
 567                 os.unlink(dest)
 568                 editedFiles.add(dest)
 569                 filesToDelete.add(src)
 570             else:
 571                 die("unknown modifier %s for %s" % (modifier, path))
 572
 573         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 574         patchcmd = diffcmd + " | git apply "
 575         tryPatchCmd = patchcmd + "--check -"
 576         applyPatchCmd = patchcmd + "--check --apply -"
 577
 578         if os.system(tryPatchCmd) != 0:
 579             print "Unfortunately applying the change failed!"
 580             print "What do you want to do?"
 581             response = "x"
 582             while response != "s" and response != "a" and response != "w":
 583                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 584                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 585             if response == "s":
 586                 print "Skipping! Good luck with the next patches..."
 587                 for f in editedFiles:
 588                     system("p4 revert \"%s\"" % f);
 589                 for f in filesToAdd:
 590                     system("rm %s" %f)
 591                 return
 592             elif response == "a":
 593                 os.system(applyPatchCmd)
 594                 if len(filesToAdd) > 0:
 595                     print "You may also want to call p4 add on the following files:"
 596                     print " ".join(filesToAdd)
 597                 if len(filesToDelete):
 598                     print "The following files should be scheduled for deletion with p4 delete:"
 599                     print " ".join(filesToDelete)
 600                 die("Please resolve and submit the conflict manually and "
 601                     + "continue afterwards with git-p4 submit --continue")
 602             elif response == "w":
 603                 system(diffcmd + " > patch.txt")
 604                 print "Patch saved to patch.txt in %s !" % self.clientPath
 605                 die("Please resolve and submit the conflict manually and "
 606                     "continue afterwards with git-p4 submit --continue")
 607
 608         system(applyPatchCmd)
 609
 610         for f in filesToAdd:
 611             system("p4 add \"%s\"" % f)
 612         for f in filesToDelete:
 613             system("p4 revert \"%s\"" % f)
 614             system("p4 delete \"%s\"" % f)
 615
 616         # Set/clear executable bits
 617         for f in filesToChangeExecBit.keys():
 618             mode = filesToChangeExecBit[f]
 619             setP4ExecBit(f, mode)
 620
 621         logMessage = extractLogMessageFromGitCommit(id)
 622         if self.isWindows:
 623             logMessage = logMessage.replace("\n", "\r\n")
 624         logMessage = logMessage.strip()
 625
 626         template = self.prepareSubmitTemplate()
 627
 628         if self.interactive:
 629             submitTemplate = self.prepareLogMessage(template, logMessage)
 630             diff = read_pipe("p4 diff -du ...")
 631
 632             for newFile in filesToAdd:
 633                 diff += "==== new file ====\n"
 634                 diff += "--- /dev/null\n"
 635                 diff += "+++ %s\n" % newFile
 636                 f = open(newFile, "r")
 637                 for line in f.readlines():
 638                     diff += "+" + line
 639                 f.close()
 640
 641             separatorLine = "######## everything below this line is just the diff #######"
 642             if platform.system() == "Windows":
 643                 separatorLine += "\r"
 644             separatorLine += "\n"
 645
 646             [handle, fileName] = tempfile.mkstemp()
 647             tmpFile = os.fdopen(handle, "w+")
 648             tmpFile.write(submitTemplate + separatorLine + diff)
 649             tmpFile.close()
 650             defaultEditor = "vi"
 651             if platform.system() == "Windows":
 652                 defaultEditor = "notepad"
 653             editor = os.environ.get("EDITOR", defaultEditor);
 654             system(editor + " " + fileName)
 655             tmpFile = open(fileName, "rb")
 656             message = tmpFile.read()
 657             tmpFile.close()
 658             os.remove(fileName)
 659             submitTemplate = message[:message.index(separatorLine)]
 660             if self.isWindows:
 661                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 662
 663             write_pipe("p4 submit -i", submitTemplate)
 664         else:
 665             fileName = "submit.txt"
 666             file = open(fileName, "w+")
 667             file.write(self.prepareLogMessage(template, logMessage))
 668             file.close()
 669             print ("Perforce submit template written as %s. "
 670                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 671                    % (fileName, fileName))
 672
 673     def run(self, args):
 674         if len(args) == 0:
 675             self.master = currentGitBranch()
 676             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 677                 die("Detecting current git branch failed!")
 678         elif len(args) == 1:
 679             self.master = args[0]
 680         else:
 681             return False
 682
 683         [upstream, settings] = findUpstreamBranchPoint()
 684         self.depotPath = settings['depot-paths'][0]
 685         if len(self.origin) == 0:
 686             self.origin = upstream
 687
 688         if self.verbose:
 689             print "Origin branch is " + self.origin
 690
 691         if len(self.depotPath) == 0:
 692             print "Internal error: cannot locate perforce depot path from existing branches"
 693             sys.exit(128)
 694
 695         self.clientPath = p4Where(self.depotPath)
 696
 697         if len(self.clientPath) == 0:
 698             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 699             sys.exit(128)
 700
 701         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 702         self.oldWorkingDirectory = os.getcwd()
 703
 704         os.chdir(self.clientPath)
 705         print "Syncronizing p4 checkout..."
 706         system("p4 sync ...")
 707
 708         self.check()
 709
 710         commits = []
 711         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 712             commits.append(line.strip())
 713         commits.reverse()
 714
 715         while len(commits) > 0:
 716             commit = commits[0]
 717             commits = commits[1:]
 718             self.applyCommit(commit)
 719             if not self.interactive:
 720                 break
 721
 722         if len(commits) == 0:
 723             print "All changes applied!"
 724             os.chdir(self.oldWorkingDirectory)
 725
 726             sync = P4Sync()
 727             sync.run([])
 728
 729             rebase = P4Rebase()
 730             rebase.rebase()
 731
 732         return True
 733
 734 class P4Sync(Command):
 735     def __init__(self):
 736         Command.__init__(self)
 737         self.options = [
 738                 optparse.make_option("--branch", dest="branch"),
 739                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 740                 optparse.make_option("--changesfile", dest="changesFile"),
 741                 optparse.make_option("--silent", dest="silent", action="store_true"),
 742                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 743                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 744                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 745                                      help="Import into refs/heads/ , not refs/remotes"),
 746                 optparse.make_option("--max-changes", dest="maxChanges"),
 747                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 748                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 749         ]
 750         self.description = """Imports from Perforce into a git repository.\n
 751     example:
 752     //depot/my/project/ -- to import the current head
 753     //depot/my/project/@all -- to import everything
 754     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 755
 756     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 757
 758         self.usage += " //depot/path[@revRange]"
 759         self.silent = False
 760         self.createdBranches = Set()
 761         self.committedChanges = Set()
 762         self.branch = ""
 763         self.detectBranches = False
 764         self.detectLabels = False
 765         self.changesFile = ""
 766         self.syncWithOrigin = True
 767         self.verbose = False
 768         self.importIntoRemotes = True
 769         self.maxChanges = ""
 770         self.isWindows = (platform.system() == "Windows")
 771         self.keepRepoPath = False
 772         self.depotPaths = None
 773         self.p4BranchesInGit = []
 774         self.cloneExclude = []
 775
 776         if gitConfig("git-p4.syncFromOrigin") == "false":
 777             self.syncWithOrigin = False
 778
 779     def extractFilesFromCommit(self, commit):
 780         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 781                              for path in self.cloneExclude]
 782         files = []
 783         fnum = 0
 784         while commit.has_key("depotFile%s" % fnum):
 785             path =  commit["depotFile%s" % fnum]
 786
 787             if [p for p in self.cloneExclude
 788                 if path.startswith (p)]:
 789                 found = False
 790             else:
 791                 found = [p for p in self.depotPaths
 792                          if path.startswith (p)]
 793             if not found:
 794                 fnum = fnum + 1
 795                 continue
 796
 797             file = {}
 798             file["path"] = path
 799             file["rev"] = commit["rev%s" % fnum]
 800             file["action"] = commit["action%s" % fnum]
 801             file["type"] = commit["type%s" % fnum]
 802             files.append(file)
 803             fnum = fnum + 1
 804         return files
 805
 806     def stripRepoPath(self, path, prefixes):
 807         if self.keepRepoPath:
 808             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 809
 810         for p in prefixes:
 811             if path.startswith(p):
 812                 path = path[len(p):]
 813
 814         return path
 815
 816     def splitFilesIntoBranches(self, commit):
 817         branches = {}
 818         fnum = 0
 819         while commit.has_key("depotFile%s" % fnum):
 820             path =  commit["depotFile%s" % fnum]
 821             found = [p for p in self.depotPaths
 822                      if path.startswith (p)]
 823             if not found:
 824                 fnum = fnum + 1
 825                 continue
 826
 827             file = {}
 828             file["path"] = path
 829             file["rev"] = commit["rev%s" % fnum]
 830             file["action"] = commit["action%s" % fnum]
 831             file["type"] = commit["type%s" % fnum]
 832             fnum = fnum + 1
 833
 834             relPath = self.stripRepoPath(path, self.depotPaths)
 835
 836             for branch in self.knownBranches.keys():
 837
 838                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 839                 if relPath.startswith(branch + "/"):
 840                     if branch not in branches:
 841                         branches[branch] = []
 842                     branches[branch].append(file)
 843                     break
 844
 845         return branches
 846
 847     ## Should move this out, doesn't use SELF.
 848     def readP4Files(self, files):
 849         files = [f for f in files
 850                  if f['action'] != 'delete']
 851
 852         if not files:
 853             return
 854
 855         filedata = p4CmdList('-x - print',
 856                              stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 857                                               for f in files]),
 858                              stdin_mode='w+')
 859         if "p4ExitCode" in filedata[0]:
 860             die("Problems executing p4. Error: [%d]."
 861                 % (filedata[0]['p4ExitCode']));
 862
 863         j = 0;
 864         contents = {}
 865         while j < len(filedata):
 866             stat = filedata[j]
 867             j += 1
 868             text = ''
 869             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 870                 tmp = filedata[j]['data']
 871                 if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 872                     tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 873                 elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 874                     tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 875                 text += tmp
 876                 j += 1
 877
 878
 879             if not stat.has_key('depotFile'):
 880                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 881                 continue
 882
 883             contents[stat['depotFile']] = text
 884
 885         for f in files:
 886             assert not f.has_key('data')
 887             f['data'] = contents[f['path']]
 888
 889     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 890         epoch = details["time"]
 891         author = details["user"]
 892
 893         if self.verbose:
 894             print "commit into %s" % branch
 895
 896         # start with reading files; if that fails, we should not
 897         # create a commit.
 898         new_files = []
 899         for f in files:
 900             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 901                 new_files.append (f)
 902             else:
 903                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 904         files = new_files
 905         self.readP4Files(files)
 906
 907
 908
 909
 910         self.gitStream.write("commit %s\n" % branch)
 911 #        gitStream.write("mark :%s\n" % details["change"])
 912         self.committedChanges.add(int(details["change"]))
 913         committer = ""
 914         if author not in self.users:
 915             self.getUserMapFromPerforceServer()
 916         if author in self.users:
 917             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 918         else:
 919             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 920
 921         self.gitStream.write("committer %s\n" % committer)
 922
 923         self.gitStream.write("data <<EOT\n")
 924         self.gitStream.write(details["desc"])
 925         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 926                              % (','.join (branchPrefixes), details["change"]))
 927         if len(details['options']) > 0:
 928             self.gitStream.write(": options = %s" % details['options'])
 929         self.gitStream.write("]\nEOT\n\n")
 930
 931         if len(parent) > 0:
 932             if self.verbose:
 933                 print "parent %s" % parent
 934             self.gitStream.write("from %s\n" % parent)
 935
 936         for file in files:
 937             if file["type"] == "apple":
 938                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 939                 continue
 940
 941             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 942             if file["action"] == "delete":
 943                 self.gitStream.write("D %s\n" % relPath)
 944             else:
 945                 data = file['data']
 946
 947                 mode = "644"
 948                 if isP4Exec(file["type"]):
 949                     mode = "755"
 950                 elif file["type"] == "symlink":
 951                     mode = "120000"
 952                     # p4 print on a symlink contains "target\n", so strip it off
 953                     data = data[:-1]
 954
 955                 if self.isWindows and file["type"].endswith("text"):
 956                     data = data.replace("\r\n", "\n")
 957
 958                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 959                 self.gitStream.write("data %s\n" % len(data))
 960                 self.gitStream.write(data)
 961                 self.gitStream.write("\n")
 962
 963         self.gitStream.write("\n")
 964
 965         change = int(details["change"])
 966
 967         if self.labels.has_key(change):
 968             label = self.labels[change]
 969             labelDetails = label[0]
 970             labelRevisions = label[1]
 971             if self.verbose:
 972                 print "Change %s is labelled %s" % (change, labelDetails)
 973
 974             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
 975                                                     for p in branchPrefixes]))
 976
 977             if len(files) == len(labelRevisions):
 978
 979                 cleanedFiles = {}
 980                 for info in files:
 981                     if info["action"] == "delete":
 982                         continue
 983                     cleanedFiles[info["depotFile"]] = info["rev"]
 984
 985                 if cleanedFiles == labelRevisions:
 986                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
 987                     self.gitStream.write("from %s\n" % branch)
 988
 989                     owner = labelDetails["Owner"]
 990                     tagger = ""
 991                     if author in self.users:
 992                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
 993                     else:
 994                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
 995                     self.gitStream.write("tagger %s\n" % tagger)
 996                     self.gitStream.write("data <<EOT\n")
 997                     self.gitStream.write(labelDetails["Description"])
 998                     self.gitStream.write("EOT\n\n")
 999
1000                 else:
1001                     if not self.silent:
1002                         print ("Tag %s does not match with change %s: files do not match."
1003                                % (labelDetails["label"], change))
1004
1005             else:
1006                 if not self.silent:
1007                     print ("Tag %s does not match with change %s: file count is different."
1008                            % (labelDetails["label"], change))
1009
1010     def getUserCacheFilename(self):
1011         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1012         return home + "/.gitp4-usercache.txt"
1013
1014     def getUserMapFromPerforceServer(self):
1015         if self.userMapFromPerforceServer:
1016             return
1017         self.users = {}
1018
1019         for output in p4CmdList("users"):
1020             if not output.has_key("User"):
1021                 continue
1022             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1023
1024
1025         s = ''
1026         for (key, val) in self.users.items():
1027             s += "%s\t%s\n" % (key, val)
1028
1029         open(self.getUserCacheFilename(), "wb").write(s)
1030         self.userMapFromPerforceServer = True
1031
1032     def loadUserMapFromCache(self):
1033         self.users = {}
1034         self.userMapFromPerforceServer = False
1035         try:
1036             cache = open(self.getUserCacheFilename(), "rb")
1037             lines = cache.readlines()
1038             cache.close()
1039             for line in lines:
1040                 entry = line.strip().split("\t")
1041                 self.users[entry[0]] = entry[1]
1042         except IOError:
1043             self.getUserMapFromPerforceServer()
1044
1045     def getLabels(self):
1046         self.labels = {}
1047
1048         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1049         if len(l) > 0 and not self.silent:
1050             print "Finding files belonging to labels in %s" % `self.depotPaths`
1051
1052         for output in l:
1053             label = output["label"]
1054             revisions = {}
1055             newestChange = 0
1056             if self.verbose:
1057                 print "Querying files for label %s" % label
1058             for file in p4CmdList("files "
1059                                   +  ' '.join (["%s...@%s" % (p, label)
1060                                                 for p in self.depotPaths])):
1061                 revisions[file["depotFile"]] = file["rev"]
1062                 change = int(file["change"])
1063                 if change > newestChange:
1064                     newestChange = change
1065
1066             self.labels[newestChange] = [output, revisions]
1067
1068         if self.verbose:
1069             print "Label changes: %s" % self.labels.keys()
1070
1071     def guessProjectName(self):
1072         for p in self.depotPaths:
1073             if p.endswith("/"):
1074                 p = p[:-1]
1075             p = p[p.strip().rfind("/") + 1:]
1076             if not p.endswith("/"):
1077                p += "/"
1078             return p
1079
1080     def getBranchMapping(self):
1081         lostAndFoundBranches = set()
1082
1083         for info in p4CmdList("branches"):
1084             details = p4Cmd("branch -o %s" % info["branch"])
1085             viewIdx = 0
1086             while details.has_key("View%s" % viewIdx):
1087                 paths = details["View%s" % viewIdx].split(" ")
1088                 viewIdx = viewIdx + 1
1089                 # require standard //depot/foo/... //depot/bar/... mapping
1090                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1091                     continue
1092                 source = paths[0]
1093                 destination = paths[1]
1094                 ## HACK
1095                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1096                     source = source[len(self.depotPaths[0]):-4]
1097                     destination = destination[len(self.depotPaths[0]):-4]
1098
1099                     if destination in self.knownBranches:
1100                         if not self.silent:
1101                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1102                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1103                         continue
1104
1105                     self.knownBranches[destination] = source
1106
1107                     lostAndFoundBranches.discard(destination)
1108
1109                     if source not in self.knownBranches:
1110                         lostAndFoundBranches.add(source)
1111
1112
1113         for branch in lostAndFoundBranches:
1114             self.knownBranches[branch] = branch
1115
1116     def getBranchMappingFromGitBranches(self):
1117         branches = p4BranchesInGit(self.importIntoRemotes)
1118         for branch in branches.keys():
1119             if branch == "master":
1120                 branch = "main"
1121             else:
1122                 branch = branch[len(self.projectName):]
1123             self.knownBranches[branch] = branch
1124
1125     def listExistingP4GitBranches(self):
1126         # branches holds mapping from name to commit
1127         branches = p4BranchesInGit(self.importIntoRemotes)
1128         self.p4BranchesInGit = branches.keys()
1129         for branch in branches.keys():
1130             self.initialParents[self.refPrefix + branch] = branches[branch]
1131
1132     def updateOptionDict(self, d):
1133         option_keys = {}
1134         if self.keepRepoPath:
1135             option_keys['keepRepoPath'] = 1
1136
1137         d["options"] = ' '.join(sorted(option_keys.keys()))
1138
1139     def readOptions(self, d):
1140         self.keepRepoPath = (d.has_key('options')
1141                              and ('keepRepoPath' in d['options']))
1142
1143     def gitRefForBranch(self, branch):
1144         if branch == "main":
1145             return self.refPrefix + "master"
1146
1147         if len(branch) <= 0:
1148             return branch
1149
1150         return self.refPrefix + self.projectName + branch
1151
1152     def gitCommitByP4Change(self, ref, change):
1153         if self.verbose:
1154             print "looking in ref " + ref + " for change %s using bisect..." % change
1155
1156         earliestCommit = ""
1157         latestCommit = parseRevision(ref)
1158
1159         while True:
1160             if self.verbose:
1161                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1162             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1163             if len(next) == 0:
1164                 if self.verbose:
1165                     print "argh"
1166                 return ""
1167             log = extractLogMessageFromGitCommit(next)
1168             settings = extractSettingsGitLog(log)
1169             currentChange = int(settings['change'])
1170             if self.verbose:
1171                 print "current change %s" % currentChange
1172
1173             if currentChange == change:
1174                 if self.verbose:
1175                     print "found %s" % next
1176                 return next
1177
1178             if currentChange < change:
1179                 earliestCommit = "^%s" % next
1180             else:
1181                 latestCommit = "%s" % next
1182
1183         return ""
1184
1185     def importNewBranch(self, branch, maxChange):
1186         # make fast-import flush all changes to disk and update the refs using the checkpoint
1187         # command so that we can try to find the branch parent in the git history
1188         self.gitStream.write("checkpoint\n\n");
1189         self.gitStream.flush();
1190         branchPrefix = self.depotPaths[0] + branch + "/"
1191         range = "@1,%s" % maxChange
1192         #print "prefix" + branchPrefix
1193         changes = p4ChangesForPaths([branchPrefix], range)
1194         if len(changes) <= 0:
1195             return False
1196         firstChange = changes[0]
1197         #print "first change in branch: %s" % firstChange
1198         sourceBranch = self.knownBranches[branch]
1199         sourceDepotPath = self.depotPaths[0] + sourceBranch
1200         sourceRef = self.gitRefForBranch(sourceBranch)
1201         #print "source " + sourceBranch
1202
1203         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1204         #print "branch parent: %s" % branchParentChange
1205         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1206         if len(gitParent) > 0:
1207             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1208             #print "parent git commit: %s" % gitParent
1209
1210         self.importChanges(changes)
1211         return True
1212
1213     def importChanges(self, changes):
1214         cnt = 1
1215         for change in changes:
1216             description = p4Cmd("describe %s" % change)
1217             self.updateOptionDict(description)
1218
1219             if not self.silent:
1220                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1221                 sys.stdout.flush()
1222             cnt = cnt + 1
1223
1224             try:
1225                 if self.detectBranches:
1226                     branches = self.splitFilesIntoBranches(description)
1227                     for branch in branches.keys():
1228                         ## HACK  --hwn
1229                         branchPrefix = self.depotPaths[0] + branch + "/"
1230
1231                         parent = ""
1232
1233                         filesForCommit = branches[branch]
1234
1235                         if self.verbose:
1236                             print "branch is %s" % branch
1237
1238                         self.updatedBranches.add(branch)
1239
1240                         if branch not in self.createdBranches:
1241                             self.createdBranches.add(branch)
1242                             parent = self.knownBranches[branch]
1243                             if parent == branch:
1244                                 parent = ""
1245                             else:
1246                                 fullBranch = self.projectName + branch
1247                                 if fullBranch not in self.p4BranchesInGit:
1248                                     if not self.silent:
1249                                         print("\n    Importing new branch %s" % fullBranch);
1250                                     if self.importNewBranch(branch, change - 1):
1251                                         parent = ""
1252                                         self.p4BranchesInGit.append(fullBranch)
1253                                     if not self.silent:
1254                                         print("\n    Resuming with change %s" % change);
1255
1256                                 if self.verbose:
1257                                     print "parent determined through known branches: %s" % parent
1258
1259                         branch = self.gitRefForBranch(branch)
1260                         parent = self.gitRefForBranch(parent)
1261
1262                         if self.verbose:
1263                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1264
1265                         if len(parent) == 0 and branch in self.initialParents:
1266                             parent = self.initialParents[branch]
1267                             del self.initialParents[branch]
1268
1269                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1270                 else:
1271                     files = self.extractFilesFromCommit(description)
1272                     self.commit(description, files, self.branch, self.depotPaths,
1273                                 self.initialParent)
1274                     self.initialParent = ""
1275             except IOError:
1276                 print self.gitError.read()
1277                 sys.exit(1)
1278
1279     def importHeadRevision(self, revision):
1280         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1281
1282         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1283         details["desc"] = ("Initial import of %s from the state at revision %s"
1284                            % (' '.join(self.depotPaths), revision))
1285         details["change"] = revision
1286         newestRevision = 0
1287
1288         fileCnt = 0
1289         for info in p4CmdList("files "
1290                               +  ' '.join(["%s...%s"
1291                                            % (p, revision)
1292                                            for p in self.depotPaths])):
1293
1294             if info['code'] == 'error':
1295                 sys.stderr.write("p4 returned an error: %s\n"
1296                                  % info['data'])
1297                 sys.exit(1)
1298
1299
1300             change = int(info["change"])
1301             if change > newestRevision:
1302                 newestRevision = change
1303
1304             if info["action"] == "delete":
1305                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1306                 #fileCnt = fileCnt + 1
1307                 continue
1308
1309             for prop in ["depotFile", "rev", "action", "type" ]:
1310                 details["%s%s" % (prop, fileCnt)] = info[prop]
1311
1312             fileCnt = fileCnt + 1
1313
1314         details["change"] = newestRevision
1315         self.updateOptionDict(details)
1316         try:
1317             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1318         except IOError:
1319             print "IO error with git fast-import. Is your git version recent enough?"
1320             print self.gitError.read()
1321
1322
1323     def run(self, args):
1324         self.depotPaths = []
1325         self.changeRange = ""
1326         self.initialParent = ""
1327         self.previousDepotPaths = []
1328
1329         # map from branch depot path to parent branch
1330         self.knownBranches = {}
1331         self.initialParents = {}
1332         self.hasOrigin = originP4BranchesExist()
1333         if not self.syncWithOrigin:
1334             self.hasOrigin = False
1335
1336         if self.importIntoRemotes:
1337             self.refPrefix = "refs/remotes/p4/"
1338         else:
1339             self.refPrefix = "refs/heads/p4/"
1340
1341         if self.syncWithOrigin and self.hasOrigin:
1342             if not self.silent:
1343                 print "Syncing with origin first by calling git fetch origin"
1344             system("git fetch origin")
1345
1346         if len(self.branch) == 0:
1347             self.branch = self.refPrefix + "master"
1348             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1349                 system("git update-ref %s refs/heads/p4" % self.branch)
1350                 system("git branch -D p4");
1351             # create it /after/ importing, when master exists
1352             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1353                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1354
1355         # TODO: should always look at previous commits,
1356         # merge with previous imports, if possible.
1357         if args == []:
1358             if self.hasOrigin:
1359                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1360             self.listExistingP4GitBranches()
1361
1362             if len(self.p4BranchesInGit) > 1:
1363                 if not self.silent:
1364                     print "Importing from/into multiple branches"
1365                 self.detectBranches = True
1366
1367             if self.verbose:
1368                 print "branches: %s" % self.p4BranchesInGit
1369
1370             p4Change = 0
1371             for branch in self.p4BranchesInGit:
1372                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1373
1374                 settings = extractSettingsGitLog(logMsg)
1375
1376                 self.readOptions(settings)
1377                 if (settings.has_key('depot-paths')
1378                     and settings.has_key ('change')):
1379                     change = int(settings['change']) + 1
1380                     p4Change = max(p4Change, change)
1381
1382                     depotPaths = sorted(settings['depot-paths'])
1383                     if self.previousDepotPaths == []:
1384                         self.previousDepotPaths = depotPaths
1385                     else:
1386                         paths = []
1387                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1388                             for i in range(0, min(len(cur), len(prev))):
1389                                 if cur[i] <> prev[i]:
1390                                     i = i - 1
1391                                     break
1392
1393                             paths.append (cur[:i + 1])
1394
1395                         self.previousDepotPaths = paths
1396
1397             if p4Change > 0:
1398                 self.depotPaths = sorted(self.previousDepotPaths)
1399                 self.changeRange = "@%s,#head" % p4Change
1400                 if not self.detectBranches:
1401                     self.initialParent = parseRevision(self.branch)
1402                 if not self.silent and not self.detectBranches:
1403                     print "Performing incremental import into %s git branch" % self.branch
1404
1405         if not self.branch.startswith("refs/"):
1406             self.branch = "refs/heads/" + self.branch
1407
1408         if len(args) == 0 and self.depotPaths:
1409             if not self.silent:
1410                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1411         else:
1412             if self.depotPaths and self.depotPaths != args:
1413                 print ("previous import used depot path %s and now %s was specified. "
1414                        "This doesn't work!" % (' '.join (self.depotPaths),
1415                                                ' '.join (args)))
1416                 sys.exit(1)
1417
1418             self.depotPaths = sorted(args)
1419
1420         revision = ""
1421         self.users = {}
1422
1423         newPaths = []
1424         for p in self.depotPaths:
1425             if p.find("@") != -1:
1426                 atIdx = p.index("@")
1427                 self.changeRange = p[atIdx:]
1428                 if self.changeRange == "@all":
1429                     self.changeRange = ""
1430                 elif ',' not in self.changeRange:
1431                     revision = self.changeRange
1432                     self.changeRange = ""
1433                 p = p[:atIdx]
1434             elif p.find("#") != -1:
1435                 hashIdx = p.index("#")
1436                 revision = p[hashIdx:]
1437                 p = p[:hashIdx]
1438             elif self.previousDepotPaths == []:
1439                 revision = "#head"
1440
1441             p = re.sub ("\.\.\.$", "", p)
1442             if not p.endswith("/"):
1443                 p += "/"
1444
1445             newPaths.append(p)
1446
1447         self.depotPaths = newPaths
1448
1449
1450         self.loadUserMapFromCache()
1451         self.labels = {}
1452         if self.detectLabels:
1453             self.getLabels();
1454
1455         if self.detectBranches:
1456             ## FIXME - what's a P4 projectName ?
1457             self.projectName = self.guessProjectName()
1458
1459             if self.hasOrigin:
1460                 self.getBranchMappingFromGitBranches()
1461             else:
1462                 self.getBranchMapping()
1463             if self.verbose:
1464                 print "p4-git branches: %s" % self.p4BranchesInGit
1465                 print "initial parents: %s" % self.initialParents
1466             for b in self.p4BranchesInGit:
1467                 if b != "master":
1468
1469                     ## FIXME
1470                     b = b[len(self.projectName):]
1471                 self.createdBranches.add(b)
1472
1473         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1474
1475         importProcess = subprocess.Popen(["git", "fast-import"],
1476                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1477                                          stderr=subprocess.PIPE);
1478         self.gitOutput = importProcess.stdout
1479         self.gitStream = importProcess.stdin
1480         self.gitError = importProcess.stderr
1481
1482         if revision:
1483             self.importHeadRevision(revision)
1484         else:
1485             changes = []
1486
1487             if len(self.changesFile) > 0:
1488                 output = open(self.changesFile).readlines()
1489                 changeSet = Set()
1490                 for line in output:
1491                     changeSet.add(int(line))
1492
1493                 for change in changeSet:
1494                     changes.append(change)
1495
1496                 changes.sort()
1497             else:
1498                 if self.verbose:
1499                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1500                                                               self.changeRange)
1501                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1502
1503                 if len(self.maxChanges) > 0:
1504                     changes = changes[:min(int(self.maxChanges), len(changes))]
1505
1506             if len(changes) == 0:
1507                 if not self.silent:
1508                     print "No changes to import!"
1509                 return True
1510
1511             if not self.silent and not self.detectBranches:
1512                 print "Import destination: %s" % self.branch
1513
1514             self.updatedBranches = set()
1515
1516             self.importChanges(changes)
1517
1518             if not self.silent:
1519                 print ""
1520                 if len(self.updatedBranches) > 0:
1521                     sys.stdout.write("Updated branches: ")
1522                     for b in self.updatedBranches:
1523                         sys.stdout.write("%s " % b)
1524                     sys.stdout.write("\n")
1525
1526         self.gitStream.close()
1527         if importProcess.wait() != 0:
1528             die("fast-import failed: %s" % self.gitError.read())
1529         self.gitOutput.close()
1530         self.gitError.close()
1531
1532         return True
1533
1534 class P4Rebase(Command):
1535     def __init__(self):
1536         Command.__init__(self)
1537         self.options = [ ]
1538         self.description = ("Fetches the latest revision from perforce and "
1539                             + "rebases the current work (branch) against it")
1540         self.verbose = False
1541
1542     def run(self, args):
1543         sync = P4Sync()
1544         sync.run([])
1545
1546         return self.rebase()
1547
1548     def rebase(self):
1549         if os.system("git update-index --refresh") != 0:
1550             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1551         if len(read_pipe("git diff-index HEAD --")) > 0:
1552             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1553
1554         [upstream, settings] = findUpstreamBranchPoint()
1555         if len(upstream) == 0:
1556             die("Cannot find upstream branchpoint for rebase")
1557
1558         # the branchpoint may be p4/foo~3, so strip off the parent
1559         upstream = re.sub("~[0-9]+$", "", upstream)
1560
1561         print "Rebasing the current branch onto %s" % upstream
1562         oldHead = read_pipe("git rev-parse HEAD").strip()
1563         system("git rebase %s" % upstream)
1564         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1565         return True
1566
1567 class P4Clone(P4Sync):
1568     def __init__(self):
1569         P4Sync.__init__(self)
1570         self.description = "Creates a new git repository and imports from Perforce into it"
1571         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1572         self.options += [
1573             optparse.make_option("--destination", dest="cloneDestination",
1574                                  action='store', default=None,
1575                                  help="where to leave result of the clone"),
1576             optparse.make_option("-/", dest="cloneExclude",
1577                                  action="append", type="string",
1578                                  help="exclude depot path")
1579         ]
1580         self.cloneDestination = None
1581         self.needsGit = False
1582
1583     # This is required for the "append" cloneExclude action
1584     def ensure_value(self, attr, value):
1585         if not hasattr(self, attr) or getattr(self, attr) is None:
1586             setattr(self, attr, value)
1587         return getattr(self, attr)
1588
1589     def defaultDestination(self, args):
1590         ## TODO: use common prefix of args?
1591         depotPath = args[0]
1592         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1593         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1594         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1595         depotDir = re.sub(r"/$", "", depotDir)
1596         return os.path.split(depotDir)[1]
1597
1598     def run(self, args):
1599         if len(args) < 1:
1600             return False
1601
1602         if self.keepRepoPath and not self.cloneDestination:
1603             sys.stderr.write("Must specify destination for --keep-path\n")
1604             sys.exit(1)
1605
1606         depotPaths = args
1607
1608         if not self.cloneDestination and len(depotPaths) > 1:
1609             self.cloneDestination = depotPaths[-1]
1610             depotPaths = depotPaths[:-1]
1611
1612         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1613         for p in depotPaths:
1614             if not p.startswith("//"):
1615                 return False
1616
1617         if not self.cloneDestination:
1618             self.cloneDestination = self.defaultDestination(args)
1619
1620         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1621         if not os.path.exists(self.cloneDestination):
1622             os.makedirs(self.cloneDestination)
1623         os.chdir(self.cloneDestination)
1624         system("git init")
1625         self.gitdir = os.getcwd() + "/.git"
1626         if not P4Sync.run(self, depotPaths):
1627             return False
1628         if self.branch != "master":
1629             if gitBranchExists("refs/remotes/p4/master"):
1630                 system("git branch master refs/remotes/p4/master")
1631                 system("git checkout -f")
1632             else:
1633                 print "Could not detect main branch. No checkout/master branch created."
1634
1635         return True
1636
1637 class P4Branches(Command):
1638     def __init__(self):
1639         Command.__init__(self)
1640         self.options = [ ]
1641         self.description = ("Shows the git branches that hold imports and their "
1642                             + "corresponding perforce depot paths")
1643         self.verbose = False
1644
1645     def run(self, args):
1646         if originP4BranchesExist():
1647             createOrUpdateBranchesFromOrigin()
1648
1649         cmdline = "git rev-parse --symbolic "
1650         cmdline += " --remotes"
1651
1652         for line in read_pipe_lines(cmdline):
1653             line = line.strip()
1654
1655             if not line.startswith('p4/') or line == "p4/HEAD":
1656                 continue
1657             branch = line
1658
1659             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1660             settings = extractSettingsGitLog(log)
1661
1662             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1663         return True
1664
1665 class HelpFormatter(optparse.IndentedHelpFormatter):
1666     def __init__(self):
1667         optparse.IndentedHelpFormatter.__init__(self)
1668
1669     def format_description(self, description):
1670         if description:
1671             return description + "\n"
1672         else:
1673             return ""
1674
1675 def printUsage(commands):
1676     print "usage: %s <command> [options]" % sys.argv[0]
1677     print ""
1678     print "valid commands: %s" % ", ".join(commands)
1679     print ""
1680     print "Try %s <command> --help for command specific help." % sys.argv[0]
1681     print ""
1682
1683 commands = {
1684     "debug" : P4Debug,
1685     "submit" : P4Submit,
1686     "commit" : P4Submit,
1687     "sync" : P4Sync,
1688     "rebase" : P4Rebase,
1689     "clone" : P4Clone,
1690     "rollback" : P4RollBack,
1691     "branches" : P4Branches
1692 }
1693
1694
1695 def main():
1696     if len(sys.argv[1:]) == 0:
1697         printUsage(commands.keys())
1698         sys.exit(2)
1699
1700     cmd = ""
1701     cmdName = sys.argv[1]
1702     try:
1703         klass = commands[cmdName]
1704         cmd = klass()
1705     except KeyError:
1706         print "unknown command %s" % cmdName
1707         print ""
1708         printUsage(commands.keys())
1709         sys.exit(2)
1710
1711     options = cmd.options
1712     cmd.gitdir = os.environ.get("GIT_DIR", None)
1713
1714     args = sys.argv[2:]
1715
1716     if len(options) > 0:
1717         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1718
1719         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1720                                        options,
1721                                        description = cmd.description,
1722                                        formatter = HelpFormatter())
1723
1724         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1725     global verbose
1726     verbose = cmd.verbose
1727     if cmd.needsGit:
1728         if cmd.gitdir == None:
1729             cmd.gitdir = os.path.abspath(".git")
1730             if not isValidGitDir(cmd.gitdir):
1731                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1732                 if os.path.exists(cmd.gitdir):
1733                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1734                     if len(cdup) > 0:
1735                         os.chdir(cdup);
1736
1737         if not isValidGitDir(cmd.gitdir):
1738             if isValidGitDir(cmd.gitdir + "/.git"):
1739                 cmd.gitdir += "/.git"
1740             else:
1741                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1742
1743         os.environ["GIT_DIR"] = cmd.gitdir
1744
1745     if not cmd.run(args):
1746         parser.print_help()
1747
1748
1749 if __name__ == '__main__':
1750     main()