contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def p4_read_pipe_lines(c):
  61     """Specifically invoke p4 on the command supplied. """
  62     real_cmd = "%s %s" % ("p4", c)
  63     if verbose:
  64         print real_cmd
  65     return read_pipe_lines(real_cmd)
  66
  67 def system(cmd):
  68     if verbose:
  69         sys.stderr.write("executing %s\n" % cmd)
  70     if os.system(cmd) != 0:
  71         die("command failed: %s" % cmd)
  72
  73 def p4_system(cmd):
  74     """Specifically invoke p4 as the system command. """
  75     real_cmd = "%s %s" % ("p4", cmd)
  76     if verbose:
  77         print real_cmd
  78     return system(real_cmd)
  79
  80 def isP4Exec(kind):
  81     """Determine if a Perforce 'kind' should have execute permission
  82
  83     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  84     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  85     a plus sign, it is also executable"""
  86     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  87
  88 def setP4ExecBit(file, mode):
  89     # Reopens an already open file and changes the execute bit to match
  90     # the execute bit setting in the passed in mode.
  91
  92     p4Type = "+x"
  93
  94     if not isModeExec(mode):
  95         p4Type = getP4OpenedType(file)
  96         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  97         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  98         if p4Type[-1] == "+":
  99             p4Type = p4Type[0:-1]
 100
 101     p4_system("reopen -t %s %s" % (p4Type, file))
 102
 103 def getP4OpenedType(file):
 104     # Returns the perforce file type for the given file.
 105
 106     result = read_pipe("p4 opened %s" % file)
 107     match = re.match(".*\((.+)\)\r?$", result)
 108     if match:
 109         return match.group(1)
 110     else:
 111         die("Could not determine file type for %s (result: '%s')" % (file, result))
 112
 113 def diffTreePattern():
 114     # This is a simple generator for the diff tree regex pattern. This could be
 115     # a class variable if this and parseDiffTreeEntry were a part of a class.
 116     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 117     while True:
 118         yield pattern
 119
 120 def parseDiffTreeEntry(entry):
 121     """Parses a single diff tree entry into its component elements.
 122
 123     See git-diff-tree(1) manpage for details about the format of the diff
 124     output. This method returns a dictionary with the following elements:
 125
 126     src_mode - The mode of the source file
 127     dst_mode - The mode of the destination file
 128     src_sha1 - The sha1 for the source file
 129     dst_sha1 - The sha1 fr the destination file
 130     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 131     status_score - The score for the status (applicable for 'C' and 'R'
 132                    statuses). This is None if there is no score.
 133     src - The path for the source file.
 134     dst - The path for the destination file. This is only present for
 135           copy or renames. If it is not present, this is None.
 136
 137     If the pattern is not matched, None is returned."""
 138
 139     match = diffTreePattern().next().match(entry)
 140     if match:
 141         return {
 142             'src_mode': match.group(1),
 143             'dst_mode': match.group(2),
 144             'src_sha1': match.group(3),
 145             'dst_sha1': match.group(4),
 146             'status': match.group(5),
 147             'status_score': match.group(6),
 148             'src': match.group(7),
 149             'dst': match.group(10)
 150         }
 151     return None
 152
 153 def isModeExec(mode):
 154     # Returns True if the given git mode represents an executable file,
 155     # otherwise False.
 156     return mode[-3:] == "755"
 157
 158 def isModeExecChanged(src_mode, dst_mode):
 159     return isModeExec(src_mode) != isModeExec(dst_mode)
 160
 161 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 162     cmd = "p4 -G %s" % cmd
 163     if verbose:
 164         sys.stderr.write("Opening pipe: %s\n" % cmd)
 165
 166     # Use a temporary file to avoid deadlocks without
 167     # subprocess.communicate(), which would put another copy
 168     # of stdout into memory.
 169     stdin_file = None
 170     if stdin is not None:
 171         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 172         stdin_file.write(stdin)
 173         stdin_file.flush()
 174         stdin_file.seek(0)
 175
 176     p4 = subprocess.Popen(cmd, shell=True,
 177                           stdin=stdin_file,
 178                           stdout=subprocess.PIPE)
 179
 180     result = []
 181     try:
 182         while True:
 183             entry = marshal.load(p4.stdout)
 184             result.append(entry)
 185     except EOFError:
 186         pass
 187     exitCode = p4.wait()
 188     if exitCode != 0:
 189         entry = {}
 190         entry["p4ExitCode"] = exitCode
 191         result.append(entry)
 192
 193     return result
 194
 195 def p4Cmd(cmd):
 196     list = p4CmdList(cmd)
 197     result = {}
 198     for entry in list:
 199         result.update(entry)
 200     return result;
 201
 202 def p4Where(depotPath):
 203     if not depotPath.endswith("/"):
 204         depotPath += "/"
 205     output = p4Cmd("where %s..." % depotPath)
 206     if output["code"] == "error":
 207         return ""
 208     clientPath = ""
 209     if "path" in output:
 210         clientPath = output.get("path")
 211     elif "data" in output:
 212         data = output.get("data")
 213         lastSpace = data.rfind(" ")
 214         clientPath = data[lastSpace + 1:]
 215
 216     if clientPath.endswith("..."):
 217         clientPath = clientPath[:-3]
 218     return clientPath
 219
 220 def currentGitBranch():
 221     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 222
 223 def isValidGitDir(path):
 224     if (os.path.exists(path + "/HEAD")
 225         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 226         return True;
 227     return False
 228
 229 def parseRevision(ref):
 230     return read_pipe("git rev-parse %s" % ref).strip()
 231
 232 def extractLogMessageFromGitCommit(commit):
 233     logMessage = ""
 234
 235     ## fixme: title is first line of commit, not 1st paragraph.
 236     foundTitle = False
 237     for log in read_pipe_lines("git cat-file commit %s" % commit):
 238        if not foundTitle:
 239            if len(log) == 1:
 240                foundTitle = True
 241            continue
 242
 243        logMessage += log
 244     return logMessage
 245
 246 def extractSettingsGitLog(log):
 247     values = {}
 248     for line in log.split("\n"):
 249         line = line.strip()
 250         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 251         if not m:
 252             continue
 253
 254         assignments = m.group(1).split (':')
 255         for a in assignments:
 256             vals = a.split ('=')
 257             key = vals[0].strip()
 258             val = ('='.join (vals[1:])).strip()
 259             if val.endswith ('\"') and val.startswith('"'):
 260                 val = val[1:-1]
 261
 262             values[key] = val
 263
 264     paths = values.get("depot-paths")
 265     if not paths:
 266         paths = values.get("depot-path")
 267     if paths:
 268         values['depot-paths'] = paths.split(',')
 269     return values
 270
 271 def gitBranchExists(branch):
 272     proc = subprocess.Popen(["git", "rev-parse", branch],
 273                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 274     return proc.wait() == 0;
 275
 276 def gitConfig(key):
 277     return read_pipe("git config %s" % key, ignore_error=True).strip()
 278
 279 def p4BranchesInGit(branchesAreInRemotes = True):
 280     branches = {}
 281
 282     cmdline = "git rev-parse --symbolic "
 283     if branchesAreInRemotes:
 284         cmdline += " --remotes"
 285     else:
 286         cmdline += " --branches"
 287
 288     for line in read_pipe_lines(cmdline):
 289         line = line.strip()
 290
 291         ## only import to p4/
 292         if not line.startswith('p4/') or line == "p4/HEAD":
 293             continue
 294         branch = line
 295
 296         # strip off p4
 297         branch = re.sub ("^p4/", "", line)
 298
 299         branches[branch] = parseRevision(line)
 300     return branches
 301
 302 def findUpstreamBranchPoint(head = "HEAD"):
 303     branches = p4BranchesInGit()
 304     # map from depot-path to branch name
 305     branchByDepotPath = {}
 306     for branch in branches.keys():
 307         tip = branches[branch]
 308         log = extractLogMessageFromGitCommit(tip)
 309         settings = extractSettingsGitLog(log)
 310         if settings.has_key("depot-paths"):
 311             paths = ",".join(settings["depot-paths"])
 312             branchByDepotPath[paths] = "remotes/p4/" + branch
 313
 314     settings = None
 315     parent = 0
 316     while parent < 65535:
 317         commit = head + "~%s" % parent
 318         log = extractLogMessageFromGitCommit(commit)
 319         settings = extractSettingsGitLog(log)
 320         if settings.has_key("depot-paths"):
 321             paths = ",".join(settings["depot-paths"])
 322             if branchByDepotPath.has_key(paths):
 323                 return [branchByDepotPath[paths], settings]
 324
 325         parent = parent + 1
 326
 327     return ["", settings]
 328
 329 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 330     if not silent:
 331         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 332                % localRefPrefix)
 333
 334     originPrefix = "origin/p4/"
 335
 336     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 337         line = line.strip()
 338         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 339             continue
 340
 341         headName = line[len(originPrefix):]
 342         remoteHead = localRefPrefix + headName
 343         originHead = line
 344
 345         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 346         if (not original.has_key('depot-paths')
 347             or not original.has_key('change')):
 348             continue
 349
 350         update = False
 351         if not gitBranchExists(remoteHead):
 352             if verbose:
 353                 print "creating %s" % remoteHead
 354             update = True
 355         else:
 356             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 357             if settings.has_key('change') > 0:
 358                 if settings['depot-paths'] == original['depot-paths']:
 359                     originP4Change = int(original['change'])
 360                     p4Change = int(settings['change'])
 361                     if originP4Change > p4Change:
 362                         print ("%s (%s) is newer than %s (%s). "
 363                                "Updating p4 branch from origin."
 364                                % (originHead, originP4Change,
 365                                   remoteHead, p4Change))
 366                         update = True
 367                 else:
 368                     print ("Ignoring: %s was imported from %s while "
 369                            "%s was imported from %s"
 370                            % (originHead, ','.join(original['depot-paths']),
 371                               remoteHead, ','.join(settings['depot-paths'])))
 372
 373         if update:
 374             system("git update-ref %s %s" % (remoteHead, originHead))
 375
 376 def originP4BranchesExist():
 377         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 378
 379 def p4ChangesForPaths(depotPaths, changeRange):
 380     assert depotPaths
 381     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 382                                                         for p in depotPaths]))
 383
 384     changes = []
 385     for line in output:
 386         changeNum = line.split(" ")[1]
 387         changes.append(int(changeNum))
 388
 389     changes.sort()
 390     return changes
 391
 392 class Command:
 393     def __init__(self):
 394         self.usage = "usage: %prog [options]"
 395         self.needsGit = True
 396
 397 class P4Debug(Command):
 398     def __init__(self):
 399         Command.__init__(self)
 400         self.options = [
 401             optparse.make_option("--verbose", dest="verbose", action="store_true",
 402                                  default=False),
 403             ]
 404         self.description = "A tool to debug the output of p4 -G."
 405         self.needsGit = False
 406         self.verbose = False
 407
 408     def run(self, args):
 409         j = 0
 410         for output in p4CmdList(" ".join(args)):
 411             print 'Element: %d' % j
 412             j += 1
 413             print output
 414         return True
 415
 416 class P4RollBack(Command):
 417     def __init__(self):
 418         Command.__init__(self)
 419         self.options = [
 420             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 421             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 422         ]
 423         self.description = "A tool to debug the multi-branch import. Don't use :)"
 424         self.verbose = False
 425         self.rollbackLocalBranches = False
 426
 427     def run(self, args):
 428         if len(args) != 1:
 429             return False
 430         maxChange = int(args[0])
 431
 432         if "p4ExitCode" in p4Cmd("changes -m 1"):
 433             die("Problems executing p4");
 434
 435         if self.rollbackLocalBranches:
 436             refPrefix = "refs/heads/"
 437             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 438         else:
 439             refPrefix = "refs/remotes/"
 440             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 441
 442         for line in lines:
 443             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 444                 line = line.strip()
 445                 ref = refPrefix + line
 446                 log = extractLogMessageFromGitCommit(ref)
 447                 settings = extractSettingsGitLog(log)
 448
 449                 depotPaths = settings['depot-paths']
 450                 change = settings['change']
 451
 452                 changed = False
 453
 454                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 455                                                            for p in depotPaths]))) == 0:
 456                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 457                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 458                     continue
 459
 460                 while change and int(change) > maxChange:
 461                     changed = True
 462                     if self.verbose:
 463                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 464                     system("git update-ref %s \"%s^\"" % (ref, ref))
 465                     log = extractLogMessageFromGitCommit(ref)
 466                     settings =  extractSettingsGitLog(log)
 467
 468
 469                     depotPaths = settings['depot-paths']
 470                     change = settings['change']
 471
 472                 if changed:
 473                     print "%s rewound to %s" % (ref, change)
 474
 475         return True
 476
 477 class P4Submit(Command):
 478     def __init__(self):
 479         Command.__init__(self)
 480         self.options = [
 481                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 482                 optparse.make_option("--origin", dest="origin"),
 483                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 484         ]
 485         self.description = "Submit changes from git to the perforce depot."
 486         self.usage += " [name of git branch to submit into perforce depot]"
 487         self.interactive = True
 488         self.origin = ""
 489         self.detectRename = False
 490         self.verbose = False
 491         self.isWindows = (platform.system() == "Windows")
 492
 493     def check(self):
 494         if len(p4CmdList("opened ...")) > 0:
 495             die("You have files opened with perforce! Close them before starting the sync.")
 496
 497     # replaces everything between 'Description:' and the next P4 submit template field with the
 498     # commit message
 499     def prepareLogMessage(self, template, message):
 500         result = ""
 501
 502         inDescriptionSection = False
 503
 504         for line in template.split("\n"):
 505             if line.startswith("#"):
 506                 result += line + "\n"
 507                 continue
 508
 509             if inDescriptionSection:
 510                 if line.startswith("Files:"):
 511                     inDescriptionSection = False
 512                 else:
 513                     continue
 514             else:
 515                 if line.startswith("Description:"):
 516                     inDescriptionSection = True
 517                     line += "\n"
 518                     for messageLine in message.split("\n"):
 519                         line += "\t" + messageLine + "\n"
 520
 521             result += line + "\n"
 522
 523         return result
 524
 525     def prepareSubmitTemplate(self):
 526         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 527         template = ""
 528         inFilesSection = False
 529         for line in p4_read_pipe_lines("change -o"):
 530             if line.endswith("\r\n"):
 531                 line = line[:-2] + "\n"
 532             if inFilesSection:
 533                 if line.startswith("\t"):
 534                     # path starts and ends with a tab
 535                     path = line[1:]
 536                     lastTab = path.rfind("\t")
 537                     if lastTab != -1:
 538                         path = path[:lastTab]
 539                         if not path.startswith(self.depotPath):
 540                             continue
 541                 else:
 542                     inFilesSection = False
 543             else:
 544                 if line.startswith("Files:"):
 545                     inFilesSection = True
 546
 547             template += line
 548
 549         return template
 550
 551     def applyCommit(self, id):
 552         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 553         diffOpts = ("", "-M")[self.detectRename]
 554         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 555         filesToAdd = set()
 556         filesToDelete = set()
 557         editedFiles = set()
 558         filesToChangeExecBit = {}
 559         for line in diff:
 560             diff = parseDiffTreeEntry(line)
 561             modifier = diff['status']
 562             path = diff['src']
 563             if modifier == "M":
 564                 p4_system("edit \"%s\"" % path)
 565                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 566                     filesToChangeExecBit[path] = diff['dst_mode']
 567                 editedFiles.add(path)
 568             elif modifier == "A":
 569                 filesToAdd.add(path)
 570                 filesToChangeExecBit[path] = diff['dst_mode']
 571                 if path in filesToDelete:
 572                     filesToDelete.remove(path)
 573             elif modifier == "D":
 574                 filesToDelete.add(path)
 575                 if path in filesToAdd:
 576                     filesToAdd.remove(path)
 577             elif modifier == "R":
 578                 src, dest = diff['src'], diff['dst']
 579                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 580                 p4_system("edit \"%s\"" % (dest))
 581                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 582                     filesToChangeExecBit[dest] = diff['dst_mode']
 583                 os.unlink(dest)
 584                 editedFiles.add(dest)
 585                 filesToDelete.add(src)
 586             else:
 587                 die("unknown modifier %s for %s" % (modifier, path))
 588
 589         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 590         patchcmd = diffcmd + " | git apply "
 591         tryPatchCmd = patchcmd + "--check -"
 592         applyPatchCmd = patchcmd + "--check --apply -"
 593
 594         if os.system(tryPatchCmd) != 0:
 595             print "Unfortunately applying the change failed!"
 596             print "What do you want to do?"
 597             response = "x"
 598             while response != "s" and response != "a" and response != "w":
 599                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 600                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 601             if response == "s":
 602                 print "Skipping! Good luck with the next patches..."
 603                 for f in editedFiles:
 604                     p4_system("revert \"%s\"" % f);
 605                 for f in filesToAdd:
 606                     system("rm %s" %f)
 607                 return
 608             elif response == "a":
 609                 os.system(applyPatchCmd)
 610                 if len(filesToAdd) > 0:
 611                     print "You may also want to call p4 add on the following files:"
 612                     print " ".join(filesToAdd)
 613                 if len(filesToDelete):
 614                     print "The following files should be scheduled for deletion with p4 delete:"
 615                     print " ".join(filesToDelete)
 616                 die("Please resolve and submit the conflict manually and "
 617                     + "continue afterwards with git-p4 submit --continue")
 618             elif response == "w":
 619                 system(diffcmd + " > patch.txt")
 620                 print "Patch saved to patch.txt in %s !" % self.clientPath
 621                 die("Please resolve and submit the conflict manually and "
 622                     "continue afterwards with git-p4 submit --continue")
 623
 624         system(applyPatchCmd)
 625
 626         for f in filesToAdd:
 627             p4_system("add \"%s\"" % f)
 628         for f in filesToDelete:
 629             p4_system("revert \"%s\"" % f)
 630             p4_system("delete \"%s\"" % f)
 631
 632         # Set/clear executable bits
 633         for f in filesToChangeExecBit.keys():
 634             mode = filesToChangeExecBit[f]
 635             setP4ExecBit(f, mode)
 636
 637         logMessage = extractLogMessageFromGitCommit(id)
 638         logMessage = logMessage.strip()
 639
 640         template = self.prepareSubmitTemplate()
 641
 642         if self.interactive:
 643             submitTemplate = self.prepareLogMessage(template, logMessage)
 644             if os.environ.has_key("P4DIFF"):
 645                 del(os.environ["P4DIFF"])
 646             diff = read_pipe("p4 diff -du ...")
 647
 648             newdiff = ""
 649             for newFile in filesToAdd:
 650                 newdiff += "==== new file ====\n"
 651                 newdiff += "--- /dev/null\n"
 652                 newdiff += "+++ %s\n" % newFile
 653                 f = open(newFile, "r")
 654                 for line in f.readlines():
 655                     newdiff += "+" + line
 656                 f.close()
 657
 658             separatorLine = "######## everything below this line is just the diff #######\n"
 659
 660             [handle, fileName] = tempfile.mkstemp()
 661             tmpFile = os.fdopen(handle, "w+")
 662             if self.isWindows:
 663                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 664                 separatorLine = separatorLine.replace("\n", "\r\n")
 665                 newdiff = newdiff.replace("\n", "\r\n")
 666             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 667             tmpFile.close()
 668             defaultEditor = "vi"
 669             if platform.system() == "Windows":
 670                 defaultEditor = "notepad"
 671             if os.environ.has_key("P4EDITOR"):
 672                 editor = os.environ.get("P4EDITOR")
 673             else:
 674                 editor = os.environ.get("EDITOR", defaultEditor);
 675             system(editor + " " + fileName)
 676             tmpFile = open(fileName, "rb")
 677             message = tmpFile.read()
 678             tmpFile.close()
 679             os.remove(fileName)
 680             submitTemplate = message[:message.index(separatorLine)]
 681             if self.isWindows:
 682                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 683
 684             write_pipe("p4 submit -i", submitTemplate)
 685         else:
 686             fileName = "submit.txt"
 687             file = open(fileName, "w+")
 688             file.write(self.prepareLogMessage(template, logMessage))
 689             file.close()
 690             print ("Perforce submit template written as %s. "
 691                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 692                    % (fileName, fileName))
 693
 694     def run(self, args):
 695         if len(args) == 0:
 696             self.master = currentGitBranch()
 697             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 698                 die("Detecting current git branch failed!")
 699         elif len(args) == 1:
 700             self.master = args[0]
 701         else:
 702             return False
 703
 704         allowSubmit = gitConfig("git-p4.allowSubmit")
 705         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 706             die("%s is not in git-p4.allowSubmit" % self.master)
 707
 708         [upstream, settings] = findUpstreamBranchPoint()
 709         self.depotPath = settings['depot-paths'][0]
 710         if len(self.origin) == 0:
 711             self.origin = upstream
 712
 713         if self.verbose:
 714             print "Origin branch is " + self.origin
 715
 716         if len(self.depotPath) == 0:
 717             print "Internal error: cannot locate perforce depot path from existing branches"
 718             sys.exit(128)
 719
 720         self.clientPath = p4Where(self.depotPath)
 721
 722         if len(self.clientPath) == 0:
 723             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 724             sys.exit(128)
 725
 726         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 727         self.oldWorkingDirectory = os.getcwd()
 728
 729         os.chdir(self.clientPath)
 730         print "Syncronizing p4 checkout..."
 731         p4_system("sync ...")
 732
 733         self.check()
 734
 735         commits = []
 736         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 737             commits.append(line.strip())
 738         commits.reverse()
 739
 740         while len(commits) > 0:
 741             commit = commits[0]
 742             commits = commits[1:]
 743             self.applyCommit(commit)
 744             if not self.interactive:
 745                 break
 746
 747         if len(commits) == 0:
 748             print "All changes applied!"
 749             os.chdir(self.oldWorkingDirectory)
 750
 751             sync = P4Sync()
 752             sync.run([])
 753
 754             rebase = P4Rebase()
 755             rebase.rebase()
 756
 757         return True
 758
 759 class P4Sync(Command):
 760     def __init__(self):
 761         Command.__init__(self)
 762         self.options = [
 763                 optparse.make_option("--branch", dest="branch"),
 764                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 765                 optparse.make_option("--changesfile", dest="changesFile"),
 766                 optparse.make_option("--silent", dest="silent", action="store_true"),
 767                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 768                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 769                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 770                                      help="Import into refs/heads/ , not refs/remotes"),
 771                 optparse.make_option("--max-changes", dest="maxChanges"),
 772                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 773                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 774                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 775                                      help="Only sync files that are included in the Perforce Client Spec")
 776         ]
 777         self.description = """Imports from Perforce into a git repository.\n
 778     example:
 779     //depot/my/project/ -- to import the current head
 780     //depot/my/project/@all -- to import everything
 781     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 782
 783     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 784
 785         self.usage += " //depot/path[@revRange]"
 786         self.silent = False
 787         self.createdBranches = Set()
 788         self.committedChanges = Set()
 789         self.branch = ""
 790         self.detectBranches = False
 791         self.detectLabels = False
 792         self.changesFile = ""
 793         self.syncWithOrigin = True
 794         self.verbose = False
 795         self.importIntoRemotes = True
 796         self.maxChanges = ""
 797         self.isWindows = (platform.system() == "Windows")
 798         self.keepRepoPath = False
 799         self.depotPaths = None
 800         self.p4BranchesInGit = []
 801         self.cloneExclude = []
 802         self.useClientSpec = False
 803         self.clientSpecDirs = []
 804
 805         if gitConfig("git-p4.syncFromOrigin") == "false":
 806             self.syncWithOrigin = False
 807
 808     def extractFilesFromCommit(self, commit):
 809         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 810                              for path in self.cloneExclude]
 811         files = []
 812         fnum = 0
 813         while commit.has_key("depotFile%s" % fnum):
 814             path =  commit["depotFile%s" % fnum]
 815
 816             if [p for p in self.cloneExclude
 817                 if path.startswith (p)]:
 818                 found = False
 819             else:
 820                 found = [p for p in self.depotPaths
 821                          if path.startswith (p)]
 822             if not found:
 823                 fnum = fnum + 1
 824                 continue
 825
 826             file = {}
 827             file["path"] = path
 828             file["rev"] = commit["rev%s" % fnum]
 829             file["action"] = commit["action%s" % fnum]
 830             file["type"] = commit["type%s" % fnum]
 831             files.append(file)
 832             fnum = fnum + 1
 833         return files
 834
 835     def stripRepoPath(self, path, prefixes):
 836         if self.keepRepoPath:
 837             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 838
 839         for p in prefixes:
 840             if path.startswith(p):
 841                 path = path[len(p):]
 842
 843         return path
 844
 845     def splitFilesIntoBranches(self, commit):
 846         branches = {}
 847         fnum = 0
 848         while commit.has_key("depotFile%s" % fnum):
 849             path =  commit["depotFile%s" % fnum]
 850             found = [p for p in self.depotPaths
 851                      if path.startswith (p)]
 852             if not found:
 853                 fnum = fnum + 1
 854                 continue
 855
 856             file = {}
 857             file["path"] = path
 858             file["rev"] = commit["rev%s" % fnum]
 859             file["action"] = commit["action%s" % fnum]
 860             file["type"] = commit["type%s" % fnum]
 861             fnum = fnum + 1
 862
 863             relPath = self.stripRepoPath(path, self.depotPaths)
 864
 865             for branch in self.knownBranches.keys():
 866
 867                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 868                 if relPath.startswith(branch + "/"):
 869                     if branch not in branches:
 870                         branches[branch] = []
 871                     branches[branch].append(file)
 872                     break
 873
 874         return branches
 875
 876     ## Should move this out, doesn't use SELF.
 877     def readP4Files(self, files):
 878         filesForCommit = []
 879         filesToRead = []
 880
 881         for f in files:
 882             includeFile = True
 883             for val in self.clientSpecDirs:
 884                 if f['path'].startswith(val[0]):
 885                     if val[1] <= 0:
 886                         includeFile = False
 887                     break
 888
 889             if includeFile:
 890                 filesForCommit.append(f)
 891                 if f['action'] != 'delete':
 892                     filesToRead.append(f)
 893
 894         filedata = []
 895         if len(filesToRead) > 0:
 896             filedata = p4CmdList('-x - print',
 897                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 898                                                   for f in filesToRead]),
 899                                  stdin_mode='w+')
 900
 901             if "p4ExitCode" in filedata[0]:
 902                 die("Problems executing p4. Error: [%d]."
 903                     % (filedata[0]['p4ExitCode']));
 904
 905         j = 0;
 906         contents = {}
 907         while j < len(filedata):
 908             stat = filedata[j]
 909             j += 1
 910             text = [];
 911             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 912                 text.append(filedata[j]['data'])
 913                 j += 1
 914             text = ''.join(text)
 915
 916             if not stat.has_key('depotFile'):
 917                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 918                 continue
 919
 920             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 921                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 922             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 923                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 924
 925             contents[stat['depotFile']] = text
 926
 927         for f in filesForCommit:
 928             path = f['path']
 929             if contents.has_key(path):
 930                 f['data'] = contents[path]
 931
 932         return filesForCommit
 933
 934     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 935         epoch = details["time"]
 936         author = details["user"]
 937
 938         if self.verbose:
 939             print "commit into %s" % branch
 940
 941         # start with reading files; if that fails, we should not
 942         # create a commit.
 943         new_files = []
 944         for f in files:
 945             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 946                 new_files.append (f)
 947             else:
 948                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 949         files = self.readP4Files(new_files)
 950
 951         self.gitStream.write("commit %s\n" % branch)
 952 #        gitStream.write("mark :%s\n" % details["change"])
 953         self.committedChanges.add(int(details["change"]))
 954         committer = ""
 955         if author not in self.users:
 956             self.getUserMapFromPerforceServer()
 957         if author in self.users:
 958             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 959         else:
 960             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 961
 962         self.gitStream.write("committer %s\n" % committer)
 963
 964         self.gitStream.write("data <<EOT\n")
 965         self.gitStream.write(details["desc"])
 966         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 967                              % (','.join (branchPrefixes), details["change"]))
 968         if len(details['options']) > 0:
 969             self.gitStream.write(": options = %s" % details['options'])
 970         self.gitStream.write("]\nEOT\n\n")
 971
 972         if len(parent) > 0:
 973             if self.verbose:
 974                 print "parent %s" % parent
 975             self.gitStream.write("from %s\n" % parent)
 976
 977         for file in files:
 978             if file["type"] == "apple":
 979                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 980                 continue
 981
 982             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 983             if file["action"] == "delete":
 984                 self.gitStream.write("D %s\n" % relPath)
 985             else:
 986                 data = file['data']
 987
 988                 mode = "644"
 989                 if isP4Exec(file["type"]):
 990                     mode = "755"
 991                 elif file["type"] == "symlink":
 992                     mode = "120000"
 993                     # p4 print on a symlink contains "target\n", so strip it off
 994                     data = data[:-1]
 995
 996                 if self.isWindows and file["type"].endswith("text"):
 997                     data = data.replace("\r\n", "\n")
 998
 999                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1000                 self.gitStream.write("data %s\n" % len(data))
1001                 self.gitStream.write(data)
1002                 self.gitStream.write("\n")
1003
1004         self.gitStream.write("\n")
1005
1006         change = int(details["change"])
1007
1008         if self.labels.has_key(change):
1009             label = self.labels[change]
1010             labelDetails = label[0]
1011             labelRevisions = label[1]
1012             if self.verbose:
1013                 print "Change %s is labelled %s" % (change, labelDetails)
1014
1015             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1016                                                     for p in branchPrefixes]))
1017
1018             if len(files) == len(labelRevisions):
1019
1020                 cleanedFiles = {}
1021                 for info in files:
1022                     if info["action"] == "delete":
1023                         continue
1024                     cleanedFiles[info["depotFile"]] = info["rev"]
1025
1026                 if cleanedFiles == labelRevisions:
1027                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1028                     self.gitStream.write("from %s\n" % branch)
1029
1030                     owner = labelDetails["Owner"]
1031                     tagger = ""
1032                     if author in self.users:
1033                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1034                     else:
1035                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1036                     self.gitStream.write("tagger %s\n" % tagger)
1037                     self.gitStream.write("data <<EOT\n")
1038                     self.gitStream.write(labelDetails["Description"])
1039                     self.gitStream.write("EOT\n\n")
1040
1041                 else:
1042                     if not self.silent:
1043                         print ("Tag %s does not match with change %s: files do not match."
1044                                % (labelDetails["label"], change))
1045
1046             else:
1047                 if not self.silent:
1048                     print ("Tag %s does not match with change %s: file count is different."
1049                            % (labelDetails["label"], change))
1050
1051     def getUserCacheFilename(self):
1052         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1053         return home + "/.gitp4-usercache.txt"
1054
1055     def getUserMapFromPerforceServer(self):
1056         if self.userMapFromPerforceServer:
1057             return
1058         self.users = {}
1059
1060         for output in p4CmdList("users"):
1061             if not output.has_key("User"):
1062                 continue
1063             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1064
1065
1066         s = ''
1067         for (key, val) in self.users.items():
1068             s += "%s\t%s\n" % (key, val)
1069
1070         open(self.getUserCacheFilename(), "wb").write(s)
1071         self.userMapFromPerforceServer = True
1072
1073     def loadUserMapFromCache(self):
1074         self.users = {}
1075         self.userMapFromPerforceServer = False
1076         try:
1077             cache = open(self.getUserCacheFilename(), "rb")
1078             lines = cache.readlines()
1079             cache.close()
1080             for line in lines:
1081                 entry = line.strip().split("\t")
1082                 self.users[entry[0]] = entry[1]
1083         except IOError:
1084             self.getUserMapFromPerforceServer()
1085
1086     def getLabels(self):
1087         self.labels = {}
1088
1089         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1090         if len(l) > 0 and not self.silent:
1091             print "Finding files belonging to labels in %s" % `self.depotPaths`
1092
1093         for output in l:
1094             label = output["label"]
1095             revisions = {}
1096             newestChange = 0
1097             if self.verbose:
1098                 print "Querying files for label %s" % label
1099             for file in p4CmdList("files "
1100                                   +  ' '.join (["%s...@%s" % (p, label)
1101                                                 for p in self.depotPaths])):
1102                 revisions[file["depotFile"]] = file["rev"]
1103                 change = int(file["change"])
1104                 if change > newestChange:
1105                     newestChange = change
1106
1107             self.labels[newestChange] = [output, revisions]
1108
1109         if self.verbose:
1110             print "Label changes: %s" % self.labels.keys()
1111
1112     def guessProjectName(self):
1113         for p in self.depotPaths:
1114             if p.endswith("/"):
1115                 p = p[:-1]
1116             p = p[p.strip().rfind("/") + 1:]
1117             if not p.endswith("/"):
1118                p += "/"
1119             return p
1120
1121     def getBranchMapping(self):
1122         lostAndFoundBranches = set()
1123
1124         for info in p4CmdList("branches"):
1125             details = p4Cmd("branch -o %s" % info["branch"])
1126             viewIdx = 0
1127             while details.has_key("View%s" % viewIdx):
1128                 paths = details["View%s" % viewIdx].split(" ")
1129                 viewIdx = viewIdx + 1
1130                 # require standard //depot/foo/... //depot/bar/... mapping
1131                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1132                     continue
1133                 source = paths[0]
1134                 destination = paths[1]
1135                 ## HACK
1136                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1137                     source = source[len(self.depotPaths[0]):-4]
1138                     destination = destination[len(self.depotPaths[0]):-4]
1139
1140                     if destination in self.knownBranches:
1141                         if not self.silent:
1142                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1143                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1144                         continue
1145
1146                     self.knownBranches[destination] = source
1147
1148                     lostAndFoundBranches.discard(destination)
1149
1150                     if source not in self.knownBranches:
1151                         lostAndFoundBranches.add(source)
1152
1153
1154         for branch in lostAndFoundBranches:
1155             self.knownBranches[branch] = branch
1156
1157     def getBranchMappingFromGitBranches(self):
1158         branches = p4BranchesInGit(self.importIntoRemotes)
1159         for branch in branches.keys():
1160             if branch == "master":
1161                 branch = "main"
1162             else:
1163                 branch = branch[len(self.projectName):]
1164             self.knownBranches[branch] = branch
1165
1166     def listExistingP4GitBranches(self):
1167         # branches holds mapping from name to commit
1168         branches = p4BranchesInGit(self.importIntoRemotes)
1169         self.p4BranchesInGit = branches.keys()
1170         for branch in branches.keys():
1171             self.initialParents[self.refPrefix + branch] = branches[branch]
1172
1173     def updateOptionDict(self, d):
1174         option_keys = {}
1175         if self.keepRepoPath:
1176             option_keys['keepRepoPath'] = 1
1177
1178         d["options"] = ' '.join(sorted(option_keys.keys()))
1179
1180     def readOptions(self, d):
1181         self.keepRepoPath = (d.has_key('options')
1182                              and ('keepRepoPath' in d['options']))
1183
1184     def gitRefForBranch(self, branch):
1185         if branch == "main":
1186             return self.refPrefix + "master"
1187
1188         if len(branch) <= 0:
1189             return branch
1190
1191         return self.refPrefix + self.projectName + branch
1192
1193     def gitCommitByP4Change(self, ref, change):
1194         if self.verbose:
1195             print "looking in ref " + ref + " for change %s using bisect..." % change
1196
1197         earliestCommit = ""
1198         latestCommit = parseRevision(ref)
1199
1200         while True:
1201             if self.verbose:
1202                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1203             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1204             if len(next) == 0:
1205                 if self.verbose:
1206                     print "argh"
1207                 return ""
1208             log = extractLogMessageFromGitCommit(next)
1209             settings = extractSettingsGitLog(log)
1210             currentChange = int(settings['change'])
1211             if self.verbose:
1212                 print "current change %s" % currentChange
1213
1214             if currentChange == change:
1215                 if self.verbose:
1216                     print "found %s" % next
1217                 return next
1218
1219             if currentChange < change:
1220                 earliestCommit = "^%s" % next
1221             else:
1222                 latestCommit = "%s" % next
1223
1224         return ""
1225
1226     def importNewBranch(self, branch, maxChange):
1227         # make fast-import flush all changes to disk and update the refs using the checkpoint
1228         # command so that we can try to find the branch parent in the git history
1229         self.gitStream.write("checkpoint\n\n");
1230         self.gitStream.flush();
1231         branchPrefix = self.depotPaths[0] + branch + "/"
1232         range = "@1,%s" % maxChange
1233         #print "prefix" + branchPrefix
1234         changes = p4ChangesForPaths([branchPrefix], range)
1235         if len(changes) <= 0:
1236             return False
1237         firstChange = changes[0]
1238         #print "first change in branch: %s" % firstChange
1239         sourceBranch = self.knownBranches[branch]
1240         sourceDepotPath = self.depotPaths[0] + sourceBranch
1241         sourceRef = self.gitRefForBranch(sourceBranch)
1242         #print "source " + sourceBranch
1243
1244         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1245         #print "branch parent: %s" % branchParentChange
1246         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1247         if len(gitParent) > 0:
1248             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1249             #print "parent git commit: %s" % gitParent
1250
1251         self.importChanges(changes)
1252         return True
1253
1254     def importChanges(self, changes):
1255         cnt = 1
1256         for change in changes:
1257             description = p4Cmd("describe %s" % change)
1258             self.updateOptionDict(description)
1259
1260             if not self.silent:
1261                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1262                 sys.stdout.flush()
1263             cnt = cnt + 1
1264
1265             try:
1266                 if self.detectBranches:
1267                     branches = self.splitFilesIntoBranches(description)
1268                     for branch in branches.keys():
1269                         ## HACK  --hwn
1270                         branchPrefix = self.depotPaths[0] + branch + "/"
1271
1272                         parent = ""
1273
1274                         filesForCommit = branches[branch]
1275
1276                         if self.verbose:
1277                             print "branch is %s" % branch
1278
1279                         self.updatedBranches.add(branch)
1280
1281                         if branch not in self.createdBranches:
1282                             self.createdBranches.add(branch)
1283                             parent = self.knownBranches[branch]
1284                             if parent == branch:
1285                                 parent = ""
1286                             else:
1287                                 fullBranch = self.projectName + branch
1288                                 if fullBranch not in self.p4BranchesInGit:
1289                                     if not self.silent:
1290                                         print("\n    Importing new branch %s" % fullBranch);
1291                                     if self.importNewBranch(branch, change - 1):
1292                                         parent = ""
1293                                         self.p4BranchesInGit.append(fullBranch)
1294                                     if not self.silent:
1295                                         print("\n    Resuming with change %s" % change);
1296
1297                                 if self.verbose:
1298                                     print "parent determined through known branches: %s" % parent
1299
1300                         branch = self.gitRefForBranch(branch)
1301                         parent = self.gitRefForBranch(parent)
1302
1303                         if self.verbose:
1304                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1305
1306                         if len(parent) == 0 and branch in self.initialParents:
1307                             parent = self.initialParents[branch]
1308                             del self.initialParents[branch]
1309
1310                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1311                 else:
1312                     files = self.extractFilesFromCommit(description)
1313                     self.commit(description, files, self.branch, self.depotPaths,
1314                                 self.initialParent)
1315                     self.initialParent = ""
1316             except IOError:
1317                 print self.gitError.read()
1318                 sys.exit(1)
1319
1320     def importHeadRevision(self, revision):
1321         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1322
1323         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1324         details["desc"] = ("Initial import of %s from the state at revision %s"
1325                            % (' '.join(self.depotPaths), revision))
1326         details["change"] = revision
1327         newestRevision = 0
1328
1329         fileCnt = 0
1330         for info in p4CmdList("files "
1331                               +  ' '.join(["%s...%s"
1332                                            % (p, revision)
1333                                            for p in self.depotPaths])):
1334
1335             if info['code'] == 'error':
1336                 sys.stderr.write("p4 returned an error: %s\n"
1337                                  % info['data'])
1338                 sys.exit(1)
1339
1340
1341             change = int(info["change"])
1342             if change > newestRevision:
1343                 newestRevision = change
1344
1345             if info["action"] == "delete":
1346                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1347                 #fileCnt = fileCnt + 1
1348                 continue
1349
1350             for prop in ["depotFile", "rev", "action", "type" ]:
1351                 details["%s%s" % (prop, fileCnt)] = info[prop]
1352
1353             fileCnt = fileCnt + 1
1354
1355         details["change"] = newestRevision
1356         self.updateOptionDict(details)
1357         try:
1358             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1359         except IOError:
1360             print "IO error with git fast-import. Is your git version recent enough?"
1361             print self.gitError.read()
1362
1363
1364     def getClientSpec(self):
1365         specList = p4CmdList( "client -o" )
1366         temp = {}
1367         for entry in specList:
1368             for k,v in entry.iteritems():
1369                 if k.startswith("View"):
1370                     if v.startswith('"'):
1371                         start = 1
1372                     else:
1373                         start = 0
1374                     index = v.find("...")
1375                     v = v[start:index]
1376                     if v.startswith("-"):
1377                         v = v[1:]
1378                         temp[v] = -len(v)
1379                     else:
1380                         temp[v] = len(v)
1381         self.clientSpecDirs = temp.items()
1382         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1383
1384     def run(self, args):
1385         self.depotPaths = []
1386         self.changeRange = ""
1387         self.initialParent = ""
1388         self.previousDepotPaths = []
1389
1390         # map from branch depot path to parent branch
1391         self.knownBranches = {}
1392         self.initialParents = {}
1393         self.hasOrigin = originP4BranchesExist()
1394         if not self.syncWithOrigin:
1395             self.hasOrigin = False
1396
1397         if self.importIntoRemotes:
1398             self.refPrefix = "refs/remotes/p4/"
1399         else:
1400             self.refPrefix = "refs/heads/p4/"
1401
1402         if self.syncWithOrigin and self.hasOrigin:
1403             if not self.silent:
1404                 print "Syncing with origin first by calling git fetch origin"
1405             system("git fetch origin")
1406
1407         if len(self.branch) == 0:
1408             self.branch = self.refPrefix + "master"
1409             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1410                 system("git update-ref %s refs/heads/p4" % self.branch)
1411                 system("git branch -D p4");
1412             # create it /after/ importing, when master exists
1413             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1414                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1415
1416         if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1417             self.getClientSpec()
1418
1419         # TODO: should always look at previous commits,
1420         # merge with previous imports, if possible.
1421         if args == []:
1422             if self.hasOrigin:
1423                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1424             self.listExistingP4GitBranches()
1425
1426             if len(self.p4BranchesInGit) > 1:
1427                 if not self.silent:
1428                     print "Importing from/into multiple branches"
1429                 self.detectBranches = True
1430
1431             if self.verbose:
1432                 print "branches: %s" % self.p4BranchesInGit
1433
1434             p4Change = 0
1435             for branch in self.p4BranchesInGit:
1436                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1437
1438                 settings = extractSettingsGitLog(logMsg)
1439
1440                 self.readOptions(settings)
1441                 if (settings.has_key('depot-paths')
1442                     and settings.has_key ('change')):
1443                     change = int(settings['change']) + 1
1444                     p4Change = max(p4Change, change)
1445
1446                     depotPaths = sorted(settings['depot-paths'])
1447                     if self.previousDepotPaths == []:
1448                         self.previousDepotPaths = depotPaths
1449                     else:
1450                         paths = []
1451                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1452                             for i in range(0, min(len(cur), len(prev))):
1453                                 if cur[i] <> prev[i]:
1454                                     i = i - 1
1455                                     break
1456
1457                             paths.append (cur[:i + 1])
1458
1459                         self.previousDepotPaths = paths
1460
1461             if p4Change > 0:
1462                 self.depotPaths = sorted(self.previousDepotPaths)
1463                 self.changeRange = "@%s,#head" % p4Change
1464                 if not self.detectBranches:
1465                     self.initialParent = parseRevision(self.branch)
1466                 if not self.silent and not self.detectBranches:
1467                     print "Performing incremental import into %s git branch" % self.branch
1468
1469         if not self.branch.startswith("refs/"):
1470             self.branch = "refs/heads/" + self.branch
1471
1472         if len(args) == 0 and self.depotPaths:
1473             if not self.silent:
1474                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1475         else:
1476             if self.depotPaths and self.depotPaths != args:
1477                 print ("previous import used depot path %s and now %s was specified. "
1478                        "This doesn't work!" % (' '.join (self.depotPaths),
1479                                                ' '.join (args)))
1480                 sys.exit(1)
1481
1482             self.depotPaths = sorted(args)
1483
1484         revision = ""
1485         self.users = {}
1486
1487         newPaths = []
1488         for p in self.depotPaths:
1489             if p.find("@") != -1:
1490                 atIdx = p.index("@")
1491                 self.changeRange = p[atIdx:]
1492                 if self.changeRange == "@all":
1493                     self.changeRange = ""
1494                 elif ',' not in self.changeRange:
1495                     revision = self.changeRange
1496                     self.changeRange = ""
1497                 p = p[:atIdx]
1498             elif p.find("#") != -1:
1499                 hashIdx = p.index("#")
1500                 revision = p[hashIdx:]
1501                 p = p[:hashIdx]
1502             elif self.previousDepotPaths == []:
1503                 revision = "#head"
1504
1505             p = re.sub ("\.\.\.$", "", p)
1506             if not p.endswith("/"):
1507                 p += "/"
1508
1509             newPaths.append(p)
1510
1511         self.depotPaths = newPaths
1512
1513
1514         self.loadUserMapFromCache()
1515         self.labels = {}
1516         if self.detectLabels:
1517             self.getLabels();
1518
1519         if self.detectBranches:
1520             ## FIXME - what's a P4 projectName ?
1521             self.projectName = self.guessProjectName()
1522
1523             if self.hasOrigin:
1524                 self.getBranchMappingFromGitBranches()
1525             else:
1526                 self.getBranchMapping()
1527             if self.verbose:
1528                 print "p4-git branches: %s" % self.p4BranchesInGit
1529                 print "initial parents: %s" % self.initialParents
1530             for b in self.p4BranchesInGit:
1531                 if b != "master":
1532
1533                     ## FIXME
1534                     b = b[len(self.projectName):]
1535                 self.createdBranches.add(b)
1536
1537         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1538
1539         importProcess = subprocess.Popen(["git", "fast-import"],
1540                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1541                                          stderr=subprocess.PIPE);
1542         self.gitOutput = importProcess.stdout
1543         self.gitStream = importProcess.stdin
1544         self.gitError = importProcess.stderr
1545
1546         if revision:
1547             self.importHeadRevision(revision)
1548         else:
1549             changes = []
1550
1551             if len(self.changesFile) > 0:
1552                 output = open(self.changesFile).readlines()
1553                 changeSet = Set()
1554                 for line in output:
1555                     changeSet.add(int(line))
1556
1557                 for change in changeSet:
1558                     changes.append(change)
1559
1560                 changes.sort()
1561             else:
1562                 if self.verbose:
1563                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1564                                                               self.changeRange)
1565                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1566
1567                 if len(self.maxChanges) > 0:
1568                     changes = changes[:min(int(self.maxChanges), len(changes))]
1569
1570             if len(changes) == 0:
1571                 if not self.silent:
1572                     print "No changes to import!"
1573                 return True
1574
1575             if not self.silent and not self.detectBranches:
1576                 print "Import destination: %s" % self.branch
1577
1578             self.updatedBranches = set()
1579
1580             self.importChanges(changes)
1581
1582             if not self.silent:
1583                 print ""
1584                 if len(self.updatedBranches) > 0:
1585                     sys.stdout.write("Updated branches: ")
1586                     for b in self.updatedBranches:
1587                         sys.stdout.write("%s " % b)
1588                     sys.stdout.write("\n")
1589
1590         self.gitStream.close()
1591         if importProcess.wait() != 0:
1592             die("fast-import failed: %s" % self.gitError.read())
1593         self.gitOutput.close()
1594         self.gitError.close()
1595
1596         return True
1597
1598 class P4Rebase(Command):
1599     def __init__(self):
1600         Command.__init__(self)
1601         self.options = [ ]
1602         self.description = ("Fetches the latest revision from perforce and "
1603                             + "rebases the current work (branch) against it")
1604         self.verbose = False
1605
1606     def run(self, args):
1607         sync = P4Sync()
1608         sync.run([])
1609
1610         return self.rebase()
1611
1612     def rebase(self):
1613         if os.system("git update-index --refresh") != 0:
1614             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1615         if len(read_pipe("git diff-index HEAD --")) > 0:
1616             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1617
1618         [upstream, settings] = findUpstreamBranchPoint()
1619         if len(upstream) == 0:
1620             die("Cannot find upstream branchpoint for rebase")
1621
1622         # the branchpoint may be p4/foo~3, so strip off the parent
1623         upstream = re.sub("~[0-9]+$", "", upstream)
1624
1625         print "Rebasing the current branch onto %s" % upstream
1626         oldHead = read_pipe("git rev-parse HEAD").strip()
1627         system("git rebase %s" % upstream)
1628         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1629         return True
1630
1631 class P4Clone(P4Sync):
1632     def __init__(self):
1633         P4Sync.__init__(self)
1634         self.description = "Creates a new git repository and imports from Perforce into it"
1635         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1636         self.options += [
1637             optparse.make_option("--destination", dest="cloneDestination",
1638                                  action='store', default=None,
1639                                  help="where to leave result of the clone"),
1640             optparse.make_option("-/", dest="cloneExclude",
1641                                  action="append", type="string",
1642                                  help="exclude depot path")
1643         ]
1644         self.cloneDestination = None
1645         self.needsGit = False
1646
1647     # This is required for the "append" cloneExclude action
1648     def ensure_value(self, attr, value):
1649         if not hasattr(self, attr) or getattr(self, attr) is None:
1650             setattr(self, attr, value)
1651         return getattr(self, attr)
1652
1653     def defaultDestination(self, args):
1654         ## TODO: use common prefix of args?
1655         depotPath = args[0]
1656         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1657         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1658         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1659         depotDir = re.sub(r"/$", "", depotDir)
1660         return os.path.split(depotDir)[1]
1661
1662     def run(self, args):
1663         if len(args) < 1:
1664             return False
1665
1666         if self.keepRepoPath and not self.cloneDestination:
1667             sys.stderr.write("Must specify destination for --keep-path\n")
1668             sys.exit(1)
1669
1670         depotPaths = args
1671
1672         if not self.cloneDestination and len(depotPaths) > 1:
1673             self.cloneDestination = depotPaths[-1]
1674             depotPaths = depotPaths[:-1]
1675
1676         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1677         for p in depotPaths:
1678             if not p.startswith("//"):
1679                 return False
1680
1681         if not self.cloneDestination:
1682             self.cloneDestination = self.defaultDestination(args)
1683
1684         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1685         if not os.path.exists(self.cloneDestination):
1686             os.makedirs(self.cloneDestination)
1687         os.chdir(self.cloneDestination)
1688         system("git init")
1689         self.gitdir = os.getcwd() + "/.git"
1690         if not P4Sync.run(self, depotPaths):
1691             return False
1692         if self.branch != "master":
1693             if gitBranchExists("refs/remotes/p4/master"):
1694                 system("git branch master refs/remotes/p4/master")
1695                 system("git checkout -f")
1696             else:
1697                 print "Could not detect main branch. No checkout/master branch created."
1698
1699         return True
1700
1701 class P4Branches(Command):
1702     def __init__(self):
1703         Command.__init__(self)
1704         self.options = [ ]
1705         self.description = ("Shows the git branches that hold imports and their "
1706                             + "corresponding perforce depot paths")
1707         self.verbose = False
1708
1709     def run(self, args):
1710         if originP4BranchesExist():
1711             createOrUpdateBranchesFromOrigin()
1712
1713         cmdline = "git rev-parse --symbolic "
1714         cmdline += " --remotes"
1715
1716         for line in read_pipe_lines(cmdline):
1717             line = line.strip()
1718
1719             if not line.startswith('p4/') or line == "p4/HEAD":
1720                 continue
1721             branch = line
1722
1723             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1724             settings = extractSettingsGitLog(log)
1725
1726             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1727         return True
1728
1729 class HelpFormatter(optparse.IndentedHelpFormatter):
1730     def __init__(self):
1731         optparse.IndentedHelpFormatter.__init__(self)
1732
1733     def format_description(self, description):
1734         if description:
1735             return description + "\n"
1736         else:
1737             return ""
1738
1739 def printUsage(commands):
1740     print "usage: %s <command> [options]" % sys.argv[0]
1741     print ""
1742     print "valid commands: %s" % ", ".join(commands)
1743     print ""
1744     print "Try %s <command> --help for command specific help." % sys.argv[0]
1745     print ""
1746
1747 commands = {
1748     "debug" : P4Debug,
1749     "submit" : P4Submit,
1750     "commit" : P4Submit,
1751     "sync" : P4Sync,
1752     "rebase" : P4Rebase,
1753     "clone" : P4Clone,
1754     "rollback" : P4RollBack,
1755     "branches" : P4Branches
1756 }
1757
1758
1759 def main():
1760     if len(sys.argv[1:]) == 0:
1761         printUsage(commands.keys())
1762         sys.exit(2)
1763
1764     cmd = ""
1765     cmdName = sys.argv[1]
1766     try:
1767         klass = commands[cmdName]
1768         cmd = klass()
1769     except KeyError:
1770         print "unknown command %s" % cmdName
1771         print ""
1772         printUsage(commands.keys())
1773         sys.exit(2)
1774
1775     options = cmd.options
1776     cmd.gitdir = os.environ.get("GIT_DIR", None)
1777
1778     args = sys.argv[2:]
1779
1780     if len(options) > 0:
1781         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1782
1783         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1784                                        options,
1785                                        description = cmd.description,
1786                                        formatter = HelpFormatter())
1787
1788         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1789     global verbose
1790     verbose = cmd.verbose
1791     if cmd.needsGit:
1792         if cmd.gitdir == None:
1793             cmd.gitdir = os.path.abspath(".git")
1794             if not isValidGitDir(cmd.gitdir):
1795                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1796                 if os.path.exists(cmd.gitdir):
1797                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1798                     if len(cdup) > 0:
1799                         os.chdir(cdup);
1800
1801         if not isValidGitDir(cmd.gitdir):
1802             if isValidGitDir(cmd.gitdir + "/.git"):
1803                 cmd.gitdir += "/.git"
1804             else:
1805                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1806
1807         os.environ["GIT_DIR"] = cmd.gitdir
1808
1809     if not cmd.run(args):
1810         parser.print_help()
1811
1812
1813 if __name__ == '__main__':
1814     main()