contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s" % file)
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                 optparse.make_option("--origin", dest="origin"),
 469                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 470         ]
 471         self.description = "Submit changes from git to the perforce depot."
 472         self.usage += " [name of git branch to submit into perforce depot]"
 473         self.interactive = True
 474         self.origin = ""
 475         self.detectRename = False
 476         self.verbose = False
 477         self.isWindows = (platform.system() == "Windows")
 478
 479     def check(self):
 480         if len(p4CmdList("opened ...")) > 0:
 481             die("You have files opened with perforce! Close them before starting the sync.")
 482
 483     # replaces everything between 'Description:' and the next P4 submit template field with the
 484     # commit message
 485     def prepareLogMessage(self, template, message):
 486         result = ""
 487
 488         inDescriptionSection = False
 489
 490         for line in template.split("\n"):
 491             if line.startswith("#"):
 492                 result += line + "\n"
 493                 continue
 494
 495             if inDescriptionSection:
 496                 if line.startswith("Files:"):
 497                     inDescriptionSection = False
 498                 else:
 499                     continue
 500             else:
 501                 if line.startswith("Description:"):
 502                     inDescriptionSection = True
 503                     line += "\n"
 504                     for messageLine in message.split("\n"):
 505                         line += "\t" + messageLine + "\n"
 506
 507             result += line + "\n"
 508
 509         return result
 510
 511     def prepareSubmitTemplate(self):
 512         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513         template = ""
 514         inFilesSection = False
 515         for line in read_pipe_lines("p4 change -o"):
 516             if inFilesSection:
 517                 if line.startswith("\t"):
 518                     # path starts and ends with a tab
 519                     path = line[1:]
 520                     lastTab = path.rfind("\t")
 521                     if lastTab != -1:
 522                         path = path[:lastTab]
 523                         if not path.startswith(self.depotPath):
 524                             continue
 525                 else:
 526                     inFilesSection = False
 527             else:
 528                 if line.startswith("Files:"):
 529                     inFilesSection = True
 530
 531             template += line
 532
 533         return template
 534
 535     def applyCommit(self, id):
 536         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 537         diffOpts = ("", "-M")[self.detectRename]
 538         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 539         filesToAdd = set()
 540         filesToDelete = set()
 541         editedFiles = set()
 542         filesToChangeExecBit = {}
 543         for line in diff:
 544             diff = parseDiffTreeEntry(line)
 545             modifier = diff['status']
 546             path = diff['src']
 547             if modifier == "M":
 548                 system("p4 edit \"%s\"" % path)
 549                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 550                     filesToChangeExecBit[path] = diff['dst_mode']
 551                 editedFiles.add(path)
 552             elif modifier == "A":
 553                 filesToAdd.add(path)
 554                 filesToChangeExecBit[path] = diff['dst_mode']
 555                 if path in filesToDelete:
 556                     filesToDelete.remove(path)
 557             elif modifier == "D":
 558                 filesToDelete.add(path)
 559                 if path in filesToAdd:
 560                     filesToAdd.remove(path)
 561             elif modifier == "R":
 562                 src, dest = diff['src'], diff['dst']
 563                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 564                 system("p4 edit \"%s\"" % (dest))
 565                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 566                     filesToChangeExecBit[dest] = diff['dst_mode']
 567                 os.unlink(dest)
 568                 editedFiles.add(dest)
 569                 filesToDelete.add(src)
 570             else:
 571                 die("unknown modifier %s for %s" % (modifier, path))
 572
 573         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 574         patchcmd = diffcmd + " | git apply "
 575         tryPatchCmd = patchcmd + "--check -"
 576         applyPatchCmd = patchcmd + "--check --apply -"
 577
 578         if os.system(tryPatchCmd) != 0:
 579             print "Unfortunately applying the change failed!"
 580             print "What do you want to do?"
 581             response = "x"
 582             while response != "s" and response != "a" and response != "w":
 583                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 584                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 585             if response == "s":
 586                 print "Skipping! Good luck with the next patches..."
 587                 for f in editedFiles:
 588                     system("p4 revert \"%s\"" % f);
 589                 for f in filesToAdd:
 590                     system("rm %s" %f)
 591                 return
 592             elif response == "a":
 593                 os.system(applyPatchCmd)
 594                 if len(filesToAdd) > 0:
 595                     print "You may also want to call p4 add on the following files:"
 596                     print " ".join(filesToAdd)
 597                 if len(filesToDelete):
 598                     print "The following files should be scheduled for deletion with p4 delete:"
 599                     print " ".join(filesToDelete)
 600                 die("Please resolve and submit the conflict manually and "
 601                     + "continue afterwards with git-p4 submit --continue")
 602             elif response == "w":
 603                 system(diffcmd + " > patch.txt")
 604                 print "Patch saved to patch.txt in %s !" % self.clientPath
 605                 die("Please resolve and submit the conflict manually and "
 606                     "continue afterwards with git-p4 submit --continue")
 607
 608         system(applyPatchCmd)
 609
 610         for f in filesToAdd:
 611             system("p4 add \"%s\"" % f)
 612         for f in filesToDelete:
 613             system("p4 revert \"%s\"" % f)
 614             system("p4 delete \"%s\"" % f)
 615
 616         # Set/clear executable bits
 617         for f in filesToChangeExecBit.keys():
 618             mode = filesToChangeExecBit[f]
 619             setP4ExecBit(f, mode)
 620
 621         logMessage = extractLogMessageFromGitCommit(id)
 622         if self.isWindows:
 623             logMessage = logMessage.replace("\n", "\r\n")
 624         logMessage = logMessage.strip()
 625
 626         template = self.prepareSubmitTemplate()
 627
 628         if self.interactive:
 629             submitTemplate = self.prepareLogMessage(template, logMessage)
 630             diff = read_pipe("p4 diff -du ...")
 631
 632             for newFile in filesToAdd:
 633                 diff += "==== new file ====\n"
 634                 diff += "--- /dev/null\n"
 635                 diff += "+++ %s\n" % newFile
 636                 f = open(newFile, "r")
 637                 for line in f.readlines():
 638                     diff += "+" + line
 639                 f.close()
 640
 641             separatorLine = "######## everything below this line is just the diff #######"
 642             if platform.system() == "Windows":
 643                 separatorLine += "\r"
 644             separatorLine += "\n"
 645
 646             [handle, fileName] = tempfile.mkstemp()
 647             tmpFile = os.fdopen(handle, "w+")
 648             tmpFile.write(submitTemplate + separatorLine + diff)
 649             tmpFile.close()
 650             defaultEditor = "vi"
 651             if platform.system() == "Windows":
 652                 defaultEditor = "notepad"
 653             editor = os.environ.get("EDITOR", defaultEditor);
 654             system(editor + " " + fileName)
 655             tmpFile = open(fileName, "rb")
 656             message = tmpFile.read()
 657             tmpFile.close()
 658             os.remove(fileName)
 659             submitTemplate = message[:message.index(separatorLine)]
 660             if self.isWindows:
 661                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 662
 663             write_pipe("p4 submit -i", submitTemplate)
 664         else:
 665             fileName = "submit.txt"
 666             file = open(fileName, "w+")
 667             file.write(self.prepareLogMessage(template, logMessage))
 668             file.close()
 669             print ("Perforce submit template written as %s. "
 670                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 671                    % (fileName, fileName))
 672
 673     def run(self, args):
 674         if len(args) == 0:
 675             self.master = currentGitBranch()
 676             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 677                 die("Detecting current git branch failed!")
 678         elif len(args) == 1:
 679             self.master = args[0]
 680         else:
 681             return False
 682
 683         [upstream, settings] = findUpstreamBranchPoint()
 684         self.depotPath = settings['depot-paths'][0]
 685         if len(self.origin) == 0:
 686             self.origin = upstream
 687
 688         if self.verbose:
 689             print "Origin branch is " + self.origin
 690
 691         if len(self.depotPath) == 0:
 692             print "Internal error: cannot locate perforce depot path from existing branches"
 693             sys.exit(128)
 694
 695         self.clientPath = p4Where(self.depotPath)
 696
 697         if len(self.clientPath) == 0:
 698             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 699             sys.exit(128)
 700
 701         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 702         self.oldWorkingDirectory = os.getcwd()
 703
 704         os.chdir(self.clientPath)
 705         print "Syncronizing p4 checkout..."
 706         system("p4 sync ...")
 707
 708         self.check()
 709
 710         commits = []
 711         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 712             commits.append(line.strip())
 713         commits.reverse()
 714
 715         while len(commits) > 0:
 716             commit = commits[0]
 717             commits = commits[1:]
 718             self.applyCommit(commit)
 719             if not self.interactive:
 720                 break
 721
 722         if len(commits) == 0:
 723             print "All changes applied!"
 724             os.chdir(self.oldWorkingDirectory)
 725
 726             sync = P4Sync()
 727             sync.run([])
 728
 729             rebase = P4Rebase()
 730             rebase.rebase()
 731
 732         return True
 733
 734 class P4Sync(Command):
 735     def __init__(self):
 736         Command.__init__(self)
 737         self.options = [
 738                 optparse.make_option("--branch", dest="branch"),
 739                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 740                 optparse.make_option("--changesfile", dest="changesFile"),
 741                 optparse.make_option("--silent", dest="silent", action="store_true"),
 742                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 743                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 744                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 745                                      help="Import into refs/heads/ , not refs/remotes"),
 746                 optparse.make_option("--max-changes", dest="maxChanges"),
 747                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 748                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 749                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 750                                      help="Only sync files that are included in the Perforce Client Spec")
 751         ]
 752         self.description = """Imports from Perforce into a git repository.\n
 753     example:
 754     //depot/my/project/ -- to import the current head
 755     //depot/my/project/@all -- to import everything
 756     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 757
 758     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 759
 760         self.usage += " //depot/path[@revRange]"
 761         self.silent = False
 762         self.createdBranches = Set()
 763         self.committedChanges = Set()
 764         self.branch = ""
 765         self.detectBranches = False
 766         self.detectLabels = False
 767         self.changesFile = ""
 768         self.syncWithOrigin = True
 769         self.verbose = False
 770         self.importIntoRemotes = True
 771         self.maxChanges = ""
 772         self.isWindows = (platform.system() == "Windows")
 773         self.keepRepoPath = False
 774         self.depotPaths = None
 775         self.p4BranchesInGit = []
 776         self.cloneExclude = []
 777         self.useClientSpec = False
 778         self.clientSpecDirs = []
 779
 780         if gitConfig("git-p4.syncFromOrigin") == "false":
 781             self.syncWithOrigin = False
 782
 783     def extractFilesFromCommit(self, commit):
 784         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 785                              for path in self.cloneExclude]
 786         files = []
 787         fnum = 0
 788         while commit.has_key("depotFile%s" % fnum):
 789             path =  commit["depotFile%s" % fnum]
 790
 791             if [p for p in self.cloneExclude
 792                 if path.startswith (p)]:
 793                 found = False
 794             else:
 795                 found = [p for p in self.depotPaths
 796                          if path.startswith (p)]
 797             if not found:
 798                 fnum = fnum + 1
 799                 continue
 800
 801             file = {}
 802             file["path"] = path
 803             file["rev"] = commit["rev%s" % fnum]
 804             file["action"] = commit["action%s" % fnum]
 805             file["type"] = commit["type%s" % fnum]
 806             files.append(file)
 807             fnum = fnum + 1
 808         return files
 809
 810     def stripRepoPath(self, path, prefixes):
 811         if self.keepRepoPath:
 812             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 813
 814         for p in prefixes:
 815             if path.startswith(p):
 816                 path = path[len(p):]
 817
 818         return path
 819
 820     def splitFilesIntoBranches(self, commit):
 821         branches = {}
 822         fnum = 0
 823         while commit.has_key("depotFile%s" % fnum):
 824             path =  commit["depotFile%s" % fnum]
 825             found = [p for p in self.depotPaths
 826                      if path.startswith (p)]
 827             if not found:
 828                 fnum = fnum + 1
 829                 continue
 830
 831             file = {}
 832             file["path"] = path
 833             file["rev"] = commit["rev%s" % fnum]
 834             file["action"] = commit["action%s" % fnum]
 835             file["type"] = commit["type%s" % fnum]
 836             fnum = fnum + 1
 837
 838             relPath = self.stripRepoPath(path, self.depotPaths)
 839
 840             for branch in self.knownBranches.keys():
 841
 842                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 843                 if relPath.startswith(branch + "/"):
 844                     if branch not in branches:
 845                         branches[branch] = []
 846                     branches[branch].append(file)
 847                     break
 848
 849         return branches
 850
 851     ## Should move this out, doesn't use SELF.
 852     def readP4Files(self, files):
 853         filesForCommit = []
 854         filesToRead = []
 855
 856         for f in files:
 857             includeFile = True
 858             for val in self.clientSpecDirs:
 859                 if f['path'].startswith(val[0]):
 860                     if val[1] <= 0:
 861                         includeFile = False
 862                     break
 863
 864             if includeFile:
 865                 filesForCommit.append(f)
 866                 if f['action'] != 'delete':
 867                     filesToRead.append(f)
 868
 869         filedata = []
 870         if len(filesToRead) > 0:
 871             filedata = p4CmdList('-x - print',
 872                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 873                                                   for f in filesToRead]),
 874                                  stdin_mode='w+')
 875
 876             if "p4ExitCode" in filedata[0]:
 877                 die("Problems executing p4. Error: [%d]."
 878                     % (filedata[0]['p4ExitCode']));
 879
 880         j = 0;
 881         contents = {}
 882         while j < len(filedata):
 883             stat = filedata[j]
 884             j += 1
 885             text = [];
 886             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 887                 text.append(filedata[j]['data'])
 888                 j += 1
 889             text = ''.join(text)
 890
 891             if not stat.has_key('depotFile'):
 892                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 893                 continue
 894
 895             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 896                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 897             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 898                 text = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 899
 900             contents[stat['depotFile']] = text
 901
 902         for f in filesForCommit:
 903             path = f['path']
 904             if contents.has_key(path):
 905                 f['data'] = contents[path]
 906
 907         return filesForCommit
 908
 909     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 910         epoch = details["time"]
 911         author = details["user"]
 912
 913         if self.verbose:
 914             print "commit into %s" % branch
 915
 916         # start with reading files; if that fails, we should not
 917         # create a commit.
 918         new_files = []
 919         for f in files:
 920             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 921                 new_files.append (f)
 922             else:
 923                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 924         files = self.readP4Files(new_files)
 925
 926         self.gitStream.write("commit %s\n" % branch)
 927 #        gitStream.write("mark :%s\n" % details["change"])
 928         self.committedChanges.add(int(details["change"]))
 929         committer = ""
 930         if author not in self.users:
 931             self.getUserMapFromPerforceServer()
 932         if author in self.users:
 933             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 934         else:
 935             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 936
 937         self.gitStream.write("committer %s\n" % committer)
 938
 939         self.gitStream.write("data <<EOT\n")
 940         self.gitStream.write(details["desc"])
 941         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 942                              % (','.join (branchPrefixes), details["change"]))
 943         if len(details['options']) > 0:
 944             self.gitStream.write(": options = %s" % details['options'])
 945         self.gitStream.write("]\nEOT\n\n")
 946
 947         if len(parent) > 0:
 948             if self.verbose:
 949                 print "parent %s" % parent
 950             self.gitStream.write("from %s\n" % parent)
 951
 952         for file in files:
 953             if file["type"] == "apple":
 954                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 955                 continue
 956
 957             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 958             if file["action"] == "delete":
 959                 self.gitStream.write("D %s\n" % relPath)
 960             else:
 961                 data = file['data']
 962
 963                 mode = "644"
 964                 if isP4Exec(file["type"]):
 965                     mode = "755"
 966                 elif file["type"] == "symlink":
 967                     mode = "120000"
 968                     # p4 print on a symlink contains "target\n", so strip it off
 969                     data = data[:-1]
 970
 971                 if self.isWindows and file["type"].endswith("text"):
 972                     data = data.replace("\r\n", "\n")
 973
 974                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 975                 self.gitStream.write("data %s\n" % len(data))
 976                 self.gitStream.write(data)
 977                 self.gitStream.write("\n")
 978
 979         self.gitStream.write("\n")
 980
 981         change = int(details["change"])
 982
 983         if self.labels.has_key(change):
 984             label = self.labels[change]
 985             labelDetails = label[0]
 986             labelRevisions = label[1]
 987             if self.verbose:
 988                 print "Change %s is labelled %s" % (change, labelDetails)
 989
 990             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
 991                                                     for p in branchPrefixes]))
 992
 993             if len(files) == len(labelRevisions):
 994
 995                 cleanedFiles = {}
 996                 for info in files:
 997                     if info["action"] == "delete":
 998                         continue
 999                     cleanedFiles[info["depotFile"]] = info["rev"]
1000
1001                 if cleanedFiles == labelRevisions:
1002                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1003                     self.gitStream.write("from %s\n" % branch)
1004
1005                     owner = labelDetails["Owner"]
1006                     tagger = ""
1007                     if author in self.users:
1008                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1009                     else:
1010                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1011                     self.gitStream.write("tagger %s\n" % tagger)
1012                     self.gitStream.write("data <<EOT\n")
1013                     self.gitStream.write(labelDetails["Description"])
1014                     self.gitStream.write("EOT\n\n")
1015
1016                 else:
1017                     if not self.silent:
1018                         print ("Tag %s does not match with change %s: files do not match."
1019                                % (labelDetails["label"], change))
1020
1021             else:
1022                 if not self.silent:
1023                     print ("Tag %s does not match with change %s: file count is different."
1024                            % (labelDetails["label"], change))
1025
1026     def getUserCacheFilename(self):
1027         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1028         return home + "/.gitp4-usercache.txt"
1029
1030     def getUserMapFromPerforceServer(self):
1031         if self.userMapFromPerforceServer:
1032             return
1033         self.users = {}
1034
1035         for output in p4CmdList("users"):
1036             if not output.has_key("User"):
1037                 continue
1038             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1039
1040
1041         s = ''
1042         for (key, val) in self.users.items():
1043             s += "%s\t%s\n" % (key, val)
1044
1045         open(self.getUserCacheFilename(), "wb").write(s)
1046         self.userMapFromPerforceServer = True
1047
1048     def loadUserMapFromCache(self):
1049         self.users = {}
1050         self.userMapFromPerforceServer = False
1051         try:
1052             cache = open(self.getUserCacheFilename(), "rb")
1053             lines = cache.readlines()
1054             cache.close()
1055             for line in lines:
1056                 entry = line.strip().split("\t")
1057                 self.users[entry[0]] = entry[1]
1058         except IOError:
1059             self.getUserMapFromPerforceServer()
1060
1061     def getLabels(self):
1062         self.labels = {}
1063
1064         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1065         if len(l) > 0 and not self.silent:
1066             print "Finding files belonging to labels in %s" % `self.depotPaths`
1067
1068         for output in l:
1069             label = output["label"]
1070             revisions = {}
1071             newestChange = 0
1072             if self.verbose:
1073                 print "Querying files for label %s" % label
1074             for file in p4CmdList("files "
1075                                   +  ' '.join (["%s...@%s" % (p, label)
1076                                                 for p in self.depotPaths])):
1077                 revisions[file["depotFile"]] = file["rev"]
1078                 change = int(file["change"])
1079                 if change > newestChange:
1080                     newestChange = change
1081
1082             self.labels[newestChange] = [output, revisions]
1083
1084         if self.verbose:
1085             print "Label changes: %s" % self.labels.keys()
1086
1087     def guessProjectName(self):
1088         for p in self.depotPaths:
1089             if p.endswith("/"):
1090                 p = p[:-1]
1091             p = p[p.strip().rfind("/") + 1:]
1092             if not p.endswith("/"):
1093                p += "/"
1094             return p
1095
1096     def getBranchMapping(self):
1097         lostAndFoundBranches = set()
1098
1099         for info in p4CmdList("branches"):
1100             details = p4Cmd("branch -o %s" % info["branch"])
1101             viewIdx = 0
1102             while details.has_key("View%s" % viewIdx):
1103                 paths = details["View%s" % viewIdx].split(" ")
1104                 viewIdx = viewIdx + 1
1105                 # require standard //depot/foo/... //depot/bar/... mapping
1106                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1107                     continue
1108                 source = paths[0]
1109                 destination = paths[1]
1110                 ## HACK
1111                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1112                     source = source[len(self.depotPaths[0]):-4]
1113                     destination = destination[len(self.depotPaths[0]):-4]
1114
1115                     if destination in self.knownBranches:
1116                         if not self.silent:
1117                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1118                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1119                         continue
1120
1121                     self.knownBranches[destination] = source
1122
1123                     lostAndFoundBranches.discard(destination)
1124
1125                     if source not in self.knownBranches:
1126                         lostAndFoundBranches.add(source)
1127
1128
1129         for branch in lostAndFoundBranches:
1130             self.knownBranches[branch] = branch
1131
1132     def getBranchMappingFromGitBranches(self):
1133         branches = p4BranchesInGit(self.importIntoRemotes)
1134         for branch in branches.keys():
1135             if branch == "master":
1136                 branch = "main"
1137             else:
1138                 branch = branch[len(self.projectName):]
1139             self.knownBranches[branch] = branch
1140
1141     def listExistingP4GitBranches(self):
1142         # branches holds mapping from name to commit
1143         branches = p4BranchesInGit(self.importIntoRemotes)
1144         self.p4BranchesInGit = branches.keys()
1145         for branch in branches.keys():
1146             self.initialParents[self.refPrefix + branch] = branches[branch]
1147
1148     def updateOptionDict(self, d):
1149         option_keys = {}
1150         if self.keepRepoPath:
1151             option_keys['keepRepoPath'] = 1
1152
1153         d["options"] = ' '.join(sorted(option_keys.keys()))
1154
1155     def readOptions(self, d):
1156         self.keepRepoPath = (d.has_key('options')
1157                              and ('keepRepoPath' in d['options']))
1158
1159     def gitRefForBranch(self, branch):
1160         if branch == "main":
1161             return self.refPrefix + "master"
1162
1163         if len(branch) <= 0:
1164             return branch
1165
1166         return self.refPrefix + self.projectName + branch
1167
1168     def gitCommitByP4Change(self, ref, change):
1169         if self.verbose:
1170             print "looking in ref " + ref + " for change %s using bisect..." % change
1171
1172         earliestCommit = ""
1173         latestCommit = parseRevision(ref)
1174
1175         while True:
1176             if self.verbose:
1177                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1178             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1179             if len(next) == 0:
1180                 if self.verbose:
1181                     print "argh"
1182                 return ""
1183             log = extractLogMessageFromGitCommit(next)
1184             settings = extractSettingsGitLog(log)
1185             currentChange = int(settings['change'])
1186             if self.verbose:
1187                 print "current change %s" % currentChange
1188
1189             if currentChange == change:
1190                 if self.verbose:
1191                     print "found %s" % next
1192                 return next
1193
1194             if currentChange < change:
1195                 earliestCommit = "^%s" % next
1196             else:
1197                 latestCommit = "%s" % next
1198
1199         return ""
1200
1201     def importNewBranch(self, branch, maxChange):
1202         # make fast-import flush all changes to disk and update the refs using the checkpoint
1203         # command so that we can try to find the branch parent in the git history
1204         self.gitStream.write("checkpoint\n\n");
1205         self.gitStream.flush();
1206         branchPrefix = self.depotPaths[0] + branch + "/"
1207         range = "@1,%s" % maxChange
1208         #print "prefix" + branchPrefix
1209         changes = p4ChangesForPaths([branchPrefix], range)
1210         if len(changes) <= 0:
1211             return False
1212         firstChange = changes[0]
1213         #print "first change in branch: %s" % firstChange
1214         sourceBranch = self.knownBranches[branch]
1215         sourceDepotPath = self.depotPaths[0] + sourceBranch
1216         sourceRef = self.gitRefForBranch(sourceBranch)
1217         #print "source " + sourceBranch
1218
1219         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1220         #print "branch parent: %s" % branchParentChange
1221         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1222         if len(gitParent) > 0:
1223             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1224             #print "parent git commit: %s" % gitParent
1225
1226         self.importChanges(changes)
1227         return True
1228
1229     def importChanges(self, changes):
1230         cnt = 1
1231         for change in changes:
1232             description = p4Cmd("describe %s" % change)
1233             self.updateOptionDict(description)
1234
1235             if not self.silent:
1236                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1237                 sys.stdout.flush()
1238             cnt = cnt + 1
1239
1240             try:
1241                 if self.detectBranches:
1242                     branches = self.splitFilesIntoBranches(description)
1243                     for branch in branches.keys():
1244                         ## HACK  --hwn
1245                         branchPrefix = self.depotPaths[0] + branch + "/"
1246
1247                         parent = ""
1248
1249                         filesForCommit = branches[branch]
1250
1251                         if self.verbose:
1252                             print "branch is %s" % branch
1253
1254                         self.updatedBranches.add(branch)
1255
1256                         if branch not in self.createdBranches:
1257                             self.createdBranches.add(branch)
1258                             parent = self.knownBranches[branch]
1259                             if parent == branch:
1260                                 parent = ""
1261                             else:
1262                                 fullBranch = self.projectName + branch
1263                                 if fullBranch not in self.p4BranchesInGit:
1264                                     if not self.silent:
1265                                         print("\n    Importing new branch %s" % fullBranch);
1266                                     if self.importNewBranch(branch, change - 1):
1267                                         parent = ""
1268                                         self.p4BranchesInGit.append(fullBranch)
1269                                     if not self.silent:
1270                                         print("\n    Resuming with change %s" % change);
1271
1272                                 if self.verbose:
1273                                     print "parent determined through known branches: %s" % parent
1274
1275                         branch = self.gitRefForBranch(branch)
1276                         parent = self.gitRefForBranch(parent)
1277
1278                         if self.verbose:
1279                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1280
1281                         if len(parent) == 0 and branch in self.initialParents:
1282                             parent = self.initialParents[branch]
1283                             del self.initialParents[branch]
1284
1285                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1286                 else:
1287                     files = self.extractFilesFromCommit(description)
1288                     self.commit(description, files, self.branch, self.depotPaths,
1289                                 self.initialParent)
1290                     self.initialParent = ""
1291             except IOError:
1292                 print self.gitError.read()
1293                 sys.exit(1)
1294
1295     def importHeadRevision(self, revision):
1296         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1297
1298         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1299         details["desc"] = ("Initial import of %s from the state at revision %s"
1300                            % (' '.join(self.depotPaths), revision))
1301         details["change"] = revision
1302         newestRevision = 0
1303
1304         fileCnt = 0
1305         for info in p4CmdList("files "
1306                               +  ' '.join(["%s...%s"
1307                                            % (p, revision)
1308                                            for p in self.depotPaths])):
1309
1310             if info['code'] == 'error':
1311                 sys.stderr.write("p4 returned an error: %s\n"
1312                                  % info['data'])
1313                 sys.exit(1)
1314
1315
1316             change = int(info["change"])
1317             if change > newestRevision:
1318                 newestRevision = change
1319
1320             if info["action"] == "delete":
1321                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1322                 #fileCnt = fileCnt + 1
1323                 continue
1324
1325             for prop in ["depotFile", "rev", "action", "type" ]:
1326                 details["%s%s" % (prop, fileCnt)] = info[prop]
1327
1328             fileCnt = fileCnt + 1
1329
1330         details["change"] = newestRevision
1331         self.updateOptionDict(details)
1332         try:
1333             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1334         except IOError:
1335             print "IO error with git fast-import. Is your git version recent enough?"
1336             print self.gitError.read()
1337
1338
1339     def getClientSpec(self):
1340         specList = p4CmdList( "client -o" )
1341         temp = {}
1342         for entry in specList:
1343             for k,v in entry.iteritems():
1344                 if k.startswith("View"):
1345                     if v.startswith('"'):
1346                         start = 1
1347                     else:
1348                         start = 0
1349                     index = v.find("...")
1350                     v = v[start:index]
1351                     if v.startswith("-"):
1352                         v = v[1:]
1353                         temp[v] = -len(v)
1354                     else:
1355                         temp[v] = len(v)
1356         self.clientSpecDirs = temp.items()
1357         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1358
1359     def run(self, args):
1360         self.depotPaths = []
1361         self.changeRange = ""
1362         self.initialParent = ""
1363         self.previousDepotPaths = []
1364
1365         # map from branch depot path to parent branch
1366         self.knownBranches = {}
1367         self.initialParents = {}
1368         self.hasOrigin = originP4BranchesExist()
1369         if not self.syncWithOrigin:
1370             self.hasOrigin = False
1371
1372         if self.importIntoRemotes:
1373             self.refPrefix = "refs/remotes/p4/"
1374         else:
1375             self.refPrefix = "refs/heads/p4/"
1376
1377         if self.syncWithOrigin and self.hasOrigin:
1378             if not self.silent:
1379                 print "Syncing with origin first by calling git fetch origin"
1380             system("git fetch origin")
1381
1382         if len(self.branch) == 0:
1383             self.branch = self.refPrefix + "master"
1384             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1385                 system("git update-ref %s refs/heads/p4" % self.branch)
1386                 system("git branch -D p4");
1387             # create it /after/ importing, when master exists
1388             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1389                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1390
1391         if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1392             self.getClientSpec()
1393
1394         # TODO: should always look at previous commits,
1395         # merge with previous imports, if possible.
1396         if args == []:
1397             if self.hasOrigin:
1398                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1399             self.listExistingP4GitBranches()
1400
1401             if len(self.p4BranchesInGit) > 1:
1402                 if not self.silent:
1403                     print "Importing from/into multiple branches"
1404                 self.detectBranches = True
1405
1406             if self.verbose:
1407                 print "branches: %s" % self.p4BranchesInGit
1408
1409             p4Change = 0
1410             for branch in self.p4BranchesInGit:
1411                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1412
1413                 settings = extractSettingsGitLog(logMsg)
1414
1415                 self.readOptions(settings)
1416                 if (settings.has_key('depot-paths')
1417                     and settings.has_key ('change')):
1418                     change = int(settings['change']) + 1
1419                     p4Change = max(p4Change, change)
1420
1421                     depotPaths = sorted(settings['depot-paths'])
1422                     if self.previousDepotPaths == []:
1423                         self.previousDepotPaths = depotPaths
1424                     else:
1425                         paths = []
1426                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1427                             for i in range(0, min(len(cur), len(prev))):
1428                                 if cur[i] <> prev[i]:
1429                                     i = i - 1
1430                                     break
1431
1432                             paths.append (cur[:i + 1])
1433
1434                         self.previousDepotPaths = paths
1435
1436             if p4Change > 0:
1437                 self.depotPaths = sorted(self.previousDepotPaths)
1438                 self.changeRange = "@%s,#head" % p4Change
1439                 if not self.detectBranches:
1440                     self.initialParent = parseRevision(self.branch)
1441                 if not self.silent and not self.detectBranches:
1442                     print "Performing incremental import into %s git branch" % self.branch
1443
1444         if not self.branch.startswith("refs/"):
1445             self.branch = "refs/heads/" + self.branch
1446
1447         if len(args) == 0 and self.depotPaths:
1448             if not self.silent:
1449                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1450         else:
1451             if self.depotPaths and self.depotPaths != args:
1452                 print ("previous import used depot path %s and now %s was specified. "
1453                        "This doesn't work!" % (' '.join (self.depotPaths),
1454                                                ' '.join (args)))
1455                 sys.exit(1)
1456
1457             self.depotPaths = sorted(args)
1458
1459         revision = ""
1460         self.users = {}
1461
1462         newPaths = []
1463         for p in self.depotPaths:
1464             if p.find("@") != -1:
1465                 atIdx = p.index("@")
1466                 self.changeRange = p[atIdx:]
1467                 if self.changeRange == "@all":
1468                     self.changeRange = ""
1469                 elif ',' not in self.changeRange:
1470                     revision = self.changeRange
1471                     self.changeRange = ""
1472                 p = p[:atIdx]
1473             elif p.find("#") != -1:
1474                 hashIdx = p.index("#")
1475                 revision = p[hashIdx:]
1476                 p = p[:hashIdx]
1477             elif self.previousDepotPaths == []:
1478                 revision = "#head"
1479
1480             p = re.sub ("\.\.\.$", "", p)
1481             if not p.endswith("/"):
1482                 p += "/"
1483
1484             newPaths.append(p)
1485
1486         self.depotPaths = newPaths
1487
1488
1489         self.loadUserMapFromCache()
1490         self.labels = {}
1491         if self.detectLabels:
1492             self.getLabels();
1493
1494         if self.detectBranches:
1495             ## FIXME - what's a P4 projectName ?
1496             self.projectName = self.guessProjectName()
1497
1498             if self.hasOrigin:
1499                 self.getBranchMappingFromGitBranches()
1500             else:
1501                 self.getBranchMapping()
1502             if self.verbose:
1503                 print "p4-git branches: %s" % self.p4BranchesInGit
1504                 print "initial parents: %s" % self.initialParents
1505             for b in self.p4BranchesInGit:
1506                 if b != "master":
1507
1508                     ## FIXME
1509                     b = b[len(self.projectName):]
1510                 self.createdBranches.add(b)
1511
1512         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1513
1514         importProcess = subprocess.Popen(["git", "fast-import"],
1515                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1516                                          stderr=subprocess.PIPE);
1517         self.gitOutput = importProcess.stdout
1518         self.gitStream = importProcess.stdin
1519         self.gitError = importProcess.stderr
1520
1521         if revision:
1522             self.importHeadRevision(revision)
1523         else:
1524             changes = []
1525
1526             if len(self.changesFile) > 0:
1527                 output = open(self.changesFile).readlines()
1528                 changeSet = Set()
1529                 for line in output:
1530                     changeSet.add(int(line))
1531
1532                 for change in changeSet:
1533                     changes.append(change)
1534
1535                 changes.sort()
1536             else:
1537                 if self.verbose:
1538                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1539                                                               self.changeRange)
1540                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1541
1542                 if len(self.maxChanges) > 0:
1543                     changes = changes[:min(int(self.maxChanges), len(changes))]
1544
1545             if len(changes) == 0:
1546                 if not self.silent:
1547                     print "No changes to import!"
1548                 return True
1549
1550             if not self.silent and not self.detectBranches:
1551                 print "Import destination: %s" % self.branch
1552
1553             self.updatedBranches = set()
1554
1555             self.importChanges(changes)
1556
1557             if not self.silent:
1558                 print ""
1559                 if len(self.updatedBranches) > 0:
1560                     sys.stdout.write("Updated branches: ")
1561                     for b in self.updatedBranches:
1562                         sys.stdout.write("%s " % b)
1563                     sys.stdout.write("\n")
1564
1565         self.gitStream.close()
1566         if importProcess.wait() != 0:
1567             die("fast-import failed: %s" % self.gitError.read())
1568         self.gitOutput.close()
1569         self.gitError.close()
1570
1571         return True
1572
1573 class P4Rebase(Command):
1574     def __init__(self):
1575         Command.__init__(self)
1576         self.options = [ ]
1577         self.description = ("Fetches the latest revision from perforce and "
1578                             + "rebases the current work (branch) against it")
1579         self.verbose = False
1580
1581     def run(self, args):
1582         sync = P4Sync()
1583         sync.run([])
1584
1585         return self.rebase()
1586
1587     def rebase(self):
1588         if os.system("git update-index --refresh") != 0:
1589             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1590         if len(read_pipe("git diff-index HEAD --")) > 0:
1591             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1592
1593         [upstream, settings] = findUpstreamBranchPoint()
1594         if len(upstream) == 0:
1595             die("Cannot find upstream branchpoint for rebase")
1596
1597         # the branchpoint may be p4/foo~3, so strip off the parent
1598         upstream = re.sub("~[0-9]+$", "", upstream)
1599
1600         print "Rebasing the current branch onto %s" % upstream
1601         oldHead = read_pipe("git rev-parse HEAD").strip()
1602         system("git rebase %s" % upstream)
1603         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1604         return True
1605
1606 class P4Clone(P4Sync):
1607     def __init__(self):
1608         P4Sync.__init__(self)
1609         self.description = "Creates a new git repository and imports from Perforce into it"
1610         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1611         self.options += [
1612             optparse.make_option("--destination", dest="cloneDestination",
1613                                  action='store', default=None,
1614                                  help="where to leave result of the clone"),
1615             optparse.make_option("-/", dest="cloneExclude",
1616                                  action="append", type="string",
1617                                  help="exclude depot path")
1618         ]
1619         self.cloneDestination = None
1620         self.needsGit = False
1621
1622     # This is required for the "append" cloneExclude action
1623     def ensure_value(self, attr, value):
1624         if not hasattr(self, attr) or getattr(self, attr) is None:
1625             setattr(self, attr, value)
1626         return getattr(self, attr)
1627
1628     def defaultDestination(self, args):
1629         ## TODO: use common prefix of args?
1630         depotPath = args[0]
1631         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1632         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1633         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1634         depotDir = re.sub(r"/$", "", depotDir)
1635         return os.path.split(depotDir)[1]
1636
1637     def run(self, args):
1638         if len(args) < 1:
1639             return False
1640
1641         if self.keepRepoPath and not self.cloneDestination:
1642             sys.stderr.write("Must specify destination for --keep-path\n")
1643             sys.exit(1)
1644
1645         depotPaths = args
1646
1647         if not self.cloneDestination and len(depotPaths) > 1:
1648             self.cloneDestination = depotPaths[-1]
1649             depotPaths = depotPaths[:-1]
1650
1651         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1652         for p in depotPaths:
1653             if not p.startswith("//"):
1654                 return False
1655
1656         if not self.cloneDestination:
1657             self.cloneDestination = self.defaultDestination(args)
1658
1659         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1660         if not os.path.exists(self.cloneDestination):
1661             os.makedirs(self.cloneDestination)
1662         os.chdir(self.cloneDestination)
1663         system("git init")
1664         self.gitdir = os.getcwd() + "/.git"
1665         if not P4Sync.run(self, depotPaths):
1666             return False
1667         if self.branch != "master":
1668             if gitBranchExists("refs/remotes/p4/master"):
1669                 system("git branch master refs/remotes/p4/master")
1670                 system("git checkout -f")
1671             else:
1672                 print "Could not detect main branch. No checkout/master branch created."
1673
1674         return True
1675
1676 class P4Branches(Command):
1677     def __init__(self):
1678         Command.__init__(self)
1679         self.options = [ ]
1680         self.description = ("Shows the git branches that hold imports and their "
1681                             + "corresponding perforce depot paths")
1682         self.verbose = False
1683
1684     def run(self, args):
1685         if originP4BranchesExist():
1686             createOrUpdateBranchesFromOrigin()
1687
1688         cmdline = "git rev-parse --symbolic "
1689         cmdline += " --remotes"
1690
1691         for line in read_pipe_lines(cmdline):
1692             line = line.strip()
1693
1694             if not line.startswith('p4/') or line == "p4/HEAD":
1695                 continue
1696             branch = line
1697
1698             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1699             settings = extractSettingsGitLog(log)
1700
1701             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1702         return True
1703
1704 class HelpFormatter(optparse.IndentedHelpFormatter):
1705     def __init__(self):
1706         optparse.IndentedHelpFormatter.__init__(self)
1707
1708     def format_description(self, description):
1709         if description:
1710             return description + "\n"
1711         else:
1712             return ""
1713
1714 def printUsage(commands):
1715     print "usage: %s <command> [options]" % sys.argv[0]
1716     print ""
1717     print "valid commands: %s" % ", ".join(commands)
1718     print ""
1719     print "Try %s <command> --help for command specific help." % sys.argv[0]
1720     print ""
1721
1722 commands = {
1723     "debug" : P4Debug,
1724     "submit" : P4Submit,
1725     "commit" : P4Submit,
1726     "sync" : P4Sync,
1727     "rebase" : P4Rebase,
1728     "clone" : P4Clone,
1729     "rollback" : P4RollBack,
1730     "branches" : P4Branches
1731 }
1732
1733
1734 def main():
1735     if len(sys.argv[1:]) == 0:
1736         printUsage(commands.keys())
1737         sys.exit(2)
1738
1739     cmd = ""
1740     cmdName = sys.argv[1]
1741     try:
1742         klass = commands[cmdName]
1743         cmd = klass()
1744     except KeyError:
1745         print "unknown command %s" % cmdName
1746         print ""
1747         printUsage(commands.keys())
1748         sys.exit(2)
1749
1750     options = cmd.options
1751     cmd.gitdir = os.environ.get("GIT_DIR", None)
1752
1753     args = sys.argv[2:]
1754
1755     if len(options) > 0:
1756         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1757
1758         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1759                                        options,
1760                                        description = cmd.description,
1761                                        formatter = HelpFormatter())
1762
1763         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1764     global verbose
1765     verbose = cmd.verbose
1766     if cmd.needsGit:
1767         if cmd.gitdir == None:
1768             cmd.gitdir = os.path.abspath(".git")
1769             if not isValidGitDir(cmd.gitdir):
1770                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1771                 if os.path.exists(cmd.gitdir):
1772                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1773                     if len(cdup) > 0:
1774                         os.chdir(cdup);
1775
1776         if not isValidGitDir(cmd.gitdir):
1777             if isValidGitDir(cmd.gitdir + "/.git"):
1778                 cmd.gitdir += "/.git"
1779             else:
1780                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1781
1782         os.environ["GIT_DIR"] = cmd.gitdir
1783
1784     if not cmd.run(args):
1785         parser.print_help()
1786
1787
1788 if __name__ == '__main__':
1789     main()