contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     if os.name == 'nt':
  56         os.environ['PWD']=dir
  57     os.chdir(dir)
  58
  59 def die(msg):
  60     if verbose:
  61         raise Exception(msg)
  62     else:
  63         sys.stderr.write(msg + "\n")
  64         sys.exit(1)
  65
  66 def write_pipe(c, str):
  67     if verbose:
  68         sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70     pipe = os.popen(c, 'w')
  71     val = pipe.write(str)
  72     if pipe.close():
  73         die('Command failed: %s' % c)
  74
  75     return val
  76
  77 def p4_write_pipe(c, str):
  78     real_cmd = p4_build_cmd(c)
  79     return write_pipe(real_cmd, str)
  80
  81 def read_pipe(c, ignore_error=False):
  82     if verbose:
  83         sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85     pipe = os.popen(c, 'rb')
  86     val = pipe.read()
  87     if pipe.close() and not ignore_error:
  88         die('Command failed: %s' % c)
  89
  90     return val
  91
  92 def p4_read_pipe(c, ignore_error=False):
  93     real_cmd = p4_build_cmd(c)
  94     return read_pipe(real_cmd, ignore_error)
  95
  96 def read_pipe_lines(c):
  97     if verbose:
  98         sys.stderr.write('Reading pipe: %s\n' % c)
  99     ## todo: check return status
 100     pipe = os.popen(c, 'rb')
 101     val = pipe.readlines()
 102     if pipe.close():
 103         die('Command failed: %s' % c)
 104
 105     return val
 106
 107 def p4_read_pipe_lines(c):
 108     """Specifically invoke p4 on the command supplied. """
 109     real_cmd = p4_build_cmd(c)
 110     return read_pipe_lines(real_cmd)
 111
 112 def system(cmd):
 113     if verbose:
 114         sys.stderr.write("executing %s\n" % cmd)
 115     if os.system(cmd) != 0:
 116         die("command failed: %s" % cmd)
 117
 118 def p4_system(cmd):
 119     """Specifically invoke p4 as the system command. """
 120     real_cmd = p4_build_cmd(cmd)
 121     return system(real_cmd)
 122
 123 def isP4Exec(kind):
 124     """Determine if a Perforce 'kind' should have execute permission
 125
 126     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128     a plus sign, it is also executable"""
 129     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131 def setP4ExecBit(file, mode):
 132     # Reopens an already open file and changes the execute bit to match
 133     # the execute bit setting in the passed in mode.
 134
 135     p4Type = "+x"
 136
 137     if not isModeExec(mode):
 138         p4Type = getP4OpenedType(file)
 139         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141         if p4Type[-1] == "+":
 142             p4Type = p4Type[0:-1]
 143
 144     p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146 def getP4OpenedType(file):
 147     # Returns the perforce file type for the given file.
 148
 149     result = p4_read_pipe("opened %s" % file)
 150     match = re.match(".*\((.+)\)\r?$", result)
 151     if match:
 152         return match.group(1)
 153     else:
 154         die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156 def diffTreePattern():
 157     # This is a simple generator for the diff tree regex pattern. This could be
 158     # a class variable if this and parseDiffTreeEntry were a part of a class.
 159     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160     while True:
 161         yield pattern
 162
 163 def parseDiffTreeEntry(entry):
 164     """Parses a single diff tree entry into its component elements.
 165
 166     See git-diff-tree(1) manpage for details about the format of the diff
 167     output. This method returns a dictionary with the following elements:
 168
 169     src_mode - The mode of the source file
 170     dst_mode - The mode of the destination file
 171     src_sha1 - The sha1 for the source file
 172     dst_sha1 - The sha1 fr the destination file
 173     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174     status_score - The score for the status (applicable for 'C' and 'R'
 175                    statuses). This is None if there is no score.
 176     src - The path for the source file.
 177     dst - The path for the destination file. This is only present for
 178           copy or renames. If it is not present, this is None.
 179
 180     If the pattern is not matched, None is returned."""
 181
 182     match = diffTreePattern().next().match(entry)
 183     if match:
 184         return {
 185             'src_mode': match.group(1),
 186             'dst_mode': match.group(2),
 187             'src_sha1': match.group(3),
 188             'dst_sha1': match.group(4),
 189             'status': match.group(5),
 190             'status_score': match.group(6),
 191             'src': match.group(7),
 192             'dst': match.group(10)
 193         }
 194     return None
 195
 196 def isModeExec(mode):
 197     # Returns True if the given git mode represents an executable file,
 198     # otherwise False.
 199     return mode[-3:] == "755"
 200
 201 def isModeExecChanged(src_mode, dst_mode):
 202     return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205     cmd = p4_build_cmd("-G %s" % (cmd))
 206     if verbose:
 207         sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209     # Use a temporary file to avoid deadlocks without
 210     # subprocess.communicate(), which would put another copy
 211     # of stdout into memory.
 212     stdin_file = None
 213     if stdin is not None:
 214         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215         stdin_file.write(stdin)
 216         stdin_file.flush()
 217         stdin_file.seek(0)
 218
 219     p4 = subprocess.Popen(cmd, shell=True,
 220                           stdin=stdin_file,
 221                           stdout=subprocess.PIPE)
 222
 223     result = []
 224     try:
 225         while True:
 226             entry = marshal.load(p4.stdout)
 227             result.append(entry)
 228     except EOFError:
 229         pass
 230     exitCode = p4.wait()
 231     if exitCode != 0:
 232         entry = {}
 233         entry["p4ExitCode"] = exitCode
 234         result.append(entry)
 235
 236     return result
 237
 238 def p4Cmd(cmd):
 239     list = p4CmdList(cmd)
 240     result = {}
 241     for entry in list:
 242         result.update(entry)
 243     return result;
 244
 245 def p4Where(depotPath):
 246     if not depotPath.endswith("/"):
 247         depotPath += "/"
 248     output = p4Cmd("where %s..." % depotPath)
 249     if output["code"] == "error":
 250         return ""
 251     clientPath = ""
 252     if "path" in output:
 253         clientPath = output.get("path")
 254     elif "data" in output:
 255         data = output.get("data")
 256         lastSpace = data.rfind(" ")
 257         clientPath = data[lastSpace + 1:]
 258
 259     if clientPath.endswith("..."):
 260         clientPath = clientPath[:-3]
 261     return clientPath
 262
 263 def currentGitBranch():
 264     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 265
 266 def isValidGitDir(path):
 267     if (os.path.exists(path + "/HEAD")
 268         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 269         return True;
 270     return False
 271
 272 def parseRevision(ref):
 273     return read_pipe("git rev-parse %s" % ref).strip()
 274
 275 def extractLogMessageFromGitCommit(commit):
 276     logMessage = ""
 277
 278     ## fixme: title is first line of commit, not 1st paragraph.
 279     foundTitle = False
 280     for log in read_pipe_lines("git cat-file commit %s" % commit):
 281        if not foundTitle:
 282            if len(log) == 1:
 283                foundTitle = True
 284            continue
 285
 286        logMessage += log
 287     return logMessage
 288
 289 def extractSettingsGitLog(log):
 290     values = {}
 291     for line in log.split("\n"):
 292         line = line.strip()
 293         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 294         if not m:
 295             continue
 296
 297         assignments = m.group(1).split (':')
 298         for a in assignments:
 299             vals = a.split ('=')
 300             key = vals[0].strip()
 301             val = ('='.join (vals[1:])).strip()
 302             if val.endswith ('\"') and val.startswith('"'):
 303                 val = val[1:-1]
 304
 305             values[key] = val
 306
 307     paths = values.get("depot-paths")
 308     if not paths:
 309         paths = values.get("depot-path")
 310     if paths:
 311         values['depot-paths'] = paths.split(',')
 312     return values
 313
 314 def gitBranchExists(branch):
 315     proc = subprocess.Popen(["git", "rev-parse", branch],
 316                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 317     return proc.wait() == 0;
 318
 319 def gitConfig(key):
 320     return read_pipe("git config %s" % key, ignore_error=True).strip()
 321
 322 def p4BranchesInGit(branchesAreInRemotes = True):
 323     branches = {}
 324
 325     cmdline = "git rev-parse --symbolic "
 326     if branchesAreInRemotes:
 327         cmdline += " --remotes"
 328     else:
 329         cmdline += " --branches"
 330
 331     for line in read_pipe_lines(cmdline):
 332         line = line.strip()
 333
 334         ## only import to p4/
 335         if not line.startswith('p4/') or line == "p4/HEAD":
 336             continue
 337         branch = line
 338
 339         # strip off p4
 340         branch = re.sub ("^p4/", "", line)
 341
 342         branches[branch] = parseRevision(line)
 343     return branches
 344
 345 def findUpstreamBranchPoint(head = "HEAD"):
 346     branches = p4BranchesInGit()
 347     # map from depot-path to branch name
 348     branchByDepotPath = {}
 349     for branch in branches.keys():
 350         tip = branches[branch]
 351         log = extractLogMessageFromGitCommit(tip)
 352         settings = extractSettingsGitLog(log)
 353         if settings.has_key("depot-paths"):
 354             paths = ",".join(settings["depot-paths"])
 355             branchByDepotPath[paths] = "remotes/p4/" + branch
 356
 357     settings = None
 358     parent = 0
 359     while parent < 65535:
 360         commit = head + "~%s" % parent
 361         log = extractLogMessageFromGitCommit(commit)
 362         settings = extractSettingsGitLog(log)
 363         if settings.has_key("depot-paths"):
 364             paths = ",".join(settings["depot-paths"])
 365             if branchByDepotPath.has_key(paths):
 366                 return [branchByDepotPath[paths], settings]
 367
 368         parent = parent + 1
 369
 370     return ["", settings]
 371
 372 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 373     if not silent:
 374         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 375                % localRefPrefix)
 376
 377     originPrefix = "origin/p4/"
 378
 379     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 380         line = line.strip()
 381         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 382             continue
 383
 384         headName = line[len(originPrefix):]
 385         remoteHead = localRefPrefix + headName
 386         originHead = line
 387
 388         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 389         if (not original.has_key('depot-paths')
 390             or not original.has_key('change')):
 391             continue
 392
 393         update = False
 394         if not gitBranchExists(remoteHead):
 395             if verbose:
 396                 print "creating %s" % remoteHead
 397             update = True
 398         else:
 399             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 400             if settings.has_key('change') > 0:
 401                 if settings['depot-paths'] == original['depot-paths']:
 402                     originP4Change = int(original['change'])
 403                     p4Change = int(settings['change'])
 404                     if originP4Change > p4Change:
 405                         print ("%s (%s) is newer than %s (%s). "
 406                                "Updating p4 branch from origin."
 407                                % (originHead, originP4Change,
 408                                   remoteHead, p4Change))
 409                         update = True
 410                 else:
 411                     print ("Ignoring: %s was imported from %s while "
 412                            "%s was imported from %s"
 413                            % (originHead, ','.join(original['depot-paths']),
 414                               remoteHead, ','.join(settings['depot-paths'])))
 415
 416         if update:
 417             system("git update-ref %s %s" % (remoteHead, originHead))
 418
 419 def originP4BranchesExist():
 420         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 421
 422 def p4ChangesForPaths(depotPaths, changeRange):
 423     assert depotPaths
 424     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 425                                                         for p in depotPaths]))
 426
 427     changes = []
 428     for line in output:
 429         changeNum = line.split(" ")[1]
 430         changes.append(int(changeNum))
 431
 432     changes.sort()
 433     return changes
 434
 435 class Command:
 436     def __init__(self):
 437         self.usage = "usage: %prog [options]"
 438         self.needsGit = True
 439
 440 class P4Debug(Command):
 441     def __init__(self):
 442         Command.__init__(self)
 443         self.options = [
 444             optparse.make_option("--verbose", dest="verbose", action="store_true",
 445                                  default=False),
 446             ]
 447         self.description = "A tool to debug the output of p4 -G."
 448         self.needsGit = False
 449         self.verbose = False
 450
 451     def run(self, args):
 452         j = 0
 453         for output in p4CmdList(" ".join(args)):
 454             print 'Element: %d' % j
 455             j += 1
 456             print output
 457         return True
 458
 459 class P4RollBack(Command):
 460     def __init__(self):
 461         Command.__init__(self)
 462         self.options = [
 463             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 464             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 465         ]
 466         self.description = "A tool to debug the multi-branch import. Don't use :)"
 467         self.verbose = False
 468         self.rollbackLocalBranches = False
 469
 470     def run(self, args):
 471         if len(args) != 1:
 472             return False
 473         maxChange = int(args[0])
 474
 475         if "p4ExitCode" in p4Cmd("changes -m 1"):
 476             die("Problems executing p4");
 477
 478         if self.rollbackLocalBranches:
 479             refPrefix = "refs/heads/"
 480             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 481         else:
 482             refPrefix = "refs/remotes/"
 483             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 484
 485         for line in lines:
 486             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 487                 line = line.strip()
 488                 ref = refPrefix + line
 489                 log = extractLogMessageFromGitCommit(ref)
 490                 settings = extractSettingsGitLog(log)
 491
 492                 depotPaths = settings['depot-paths']
 493                 change = settings['change']
 494
 495                 changed = False
 496
 497                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 498                                                            for p in depotPaths]))) == 0:
 499                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 500                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 501                     continue
 502
 503                 while change and int(change) > maxChange:
 504                     changed = True
 505                     if self.verbose:
 506                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 507                     system("git update-ref %s \"%s^\"" % (ref, ref))
 508                     log = extractLogMessageFromGitCommit(ref)
 509                     settings =  extractSettingsGitLog(log)
 510
 511
 512                     depotPaths = settings['depot-paths']
 513                     change = settings['change']
 514
 515                 if changed:
 516                     print "%s rewound to %s" % (ref, change)
 517
 518         return True
 519
 520 class P4Submit(Command):
 521     def __init__(self):
 522         Command.__init__(self)
 523         self.options = [
 524                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 525                 optparse.make_option("--origin", dest="origin"),
 526                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 527         ]
 528         self.description = "Submit changes from git to the perforce depot."
 529         self.usage += " [name of git branch to submit into perforce depot]"
 530         self.interactive = True
 531         self.origin = ""
 532         self.detectRename = False
 533         self.verbose = False
 534         self.isWindows = (platform.system() == "Windows")
 535
 536     def check(self):
 537         if len(p4CmdList("opened ...")) > 0:
 538             die("You have files opened with perforce! Close them before starting the sync.")
 539
 540     # replaces everything between 'Description:' and the next P4 submit template field with the
 541     # commit message
 542     def prepareLogMessage(self, template, message):
 543         result = ""
 544
 545         inDescriptionSection = False
 546
 547         for line in template.split("\n"):
 548             if line.startswith("#"):
 549                 result += line + "\n"
 550                 continue
 551
 552             if inDescriptionSection:
 553                 if line.startswith("Files:"):
 554                     inDescriptionSection = False
 555                 else:
 556                     continue
 557             else:
 558                 if line.startswith("Description:"):
 559                     inDescriptionSection = True
 560                     line += "\n"
 561                     for messageLine in message.split("\n"):
 562                         line += "\t" + messageLine + "\n"
 563
 564             result += line + "\n"
 565
 566         return result
 567
 568     def prepareSubmitTemplate(self):
 569         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 570         template = ""
 571         inFilesSection = False
 572         for line in p4_read_pipe_lines("change -o"):
 573             if line.endswith("\r\n"):
 574                 line = line[:-2] + "\n"
 575             if inFilesSection:
 576                 if line.startswith("\t"):
 577                     # path starts and ends with a tab
 578                     path = line[1:]
 579                     lastTab = path.rfind("\t")
 580                     if lastTab != -1:
 581                         path = path[:lastTab]
 582                         if not path.startswith(self.depotPath):
 583                             continue
 584                 else:
 585                     inFilesSection = False
 586             else:
 587                 if line.startswith("Files:"):
 588                     inFilesSection = True
 589
 590             template += line
 591
 592         return template
 593
 594     def applyCommit(self, id):
 595         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 596         diffOpts = ("", "-M")[self.detectRename]
 597         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 598         filesToAdd = set()
 599         filesToDelete = set()
 600         editedFiles = set()
 601         filesToChangeExecBit = {}
 602         for line in diff:
 603             diff = parseDiffTreeEntry(line)
 604             modifier = diff['status']
 605             path = diff['src']
 606             if modifier == "M":
 607                 p4_system("edit \"%s\"" % path)
 608                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 609                     filesToChangeExecBit[path] = diff['dst_mode']
 610                 editedFiles.add(path)
 611             elif modifier == "A":
 612                 filesToAdd.add(path)
 613                 filesToChangeExecBit[path] = diff['dst_mode']
 614                 if path in filesToDelete:
 615                     filesToDelete.remove(path)
 616             elif modifier == "D":
 617                 filesToDelete.add(path)
 618                 if path in filesToAdd:
 619                     filesToAdd.remove(path)
 620             elif modifier == "R":
 621                 src, dest = diff['src'], diff['dst']
 622                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 623                 p4_system("edit \"%s\"" % (dest))
 624                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 625                     filesToChangeExecBit[dest] = diff['dst_mode']
 626                 os.unlink(dest)
 627                 editedFiles.add(dest)
 628                 filesToDelete.add(src)
 629             else:
 630                 die("unknown modifier %s for %s" % (modifier, path))
 631
 632         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 633         patchcmd = diffcmd + " | git apply "
 634         tryPatchCmd = patchcmd + "--check -"
 635         applyPatchCmd = patchcmd + "--check --apply -"
 636
 637         if os.system(tryPatchCmd) != 0:
 638             print "Unfortunately applying the change failed!"
 639             print "What do you want to do?"
 640             response = "x"
 641             while response != "s" and response != "a" and response != "w":
 642                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 643                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 644             if response == "s":
 645                 print "Skipping! Good luck with the next patches..."
 646                 for f in editedFiles:
 647                     p4_system("revert \"%s\"" % f);
 648                 for f in filesToAdd:
 649                     system("rm %s" %f)
 650                 return
 651             elif response == "a":
 652                 os.system(applyPatchCmd)
 653                 if len(filesToAdd) > 0:
 654                     print "You may also want to call p4 add on the following files:"
 655                     print " ".join(filesToAdd)
 656                 if len(filesToDelete):
 657                     print "The following files should be scheduled for deletion with p4 delete:"
 658                     print " ".join(filesToDelete)
 659                 die("Please resolve and submit the conflict manually and "
 660                     + "continue afterwards with git-p4 submit --continue")
 661             elif response == "w":
 662                 system(diffcmd + " > patch.txt")
 663                 print "Patch saved to patch.txt in %s !" % self.clientPath
 664                 die("Please resolve and submit the conflict manually and "
 665                     "continue afterwards with git-p4 submit --continue")
 666
 667         system(applyPatchCmd)
 668
 669         for f in filesToAdd:
 670             p4_system("add \"%s\"" % f)
 671         for f in filesToDelete:
 672             p4_system("revert \"%s\"" % f)
 673             p4_system("delete \"%s\"" % f)
 674
 675         # Set/clear executable bits
 676         for f in filesToChangeExecBit.keys():
 677             mode = filesToChangeExecBit[f]
 678             setP4ExecBit(f, mode)
 679
 680         logMessage = extractLogMessageFromGitCommit(id)
 681         logMessage = logMessage.strip()
 682
 683         template = self.prepareSubmitTemplate()
 684
 685         if self.interactive:
 686             submitTemplate = self.prepareLogMessage(template, logMessage)
 687             if os.environ.has_key("P4DIFF"):
 688                 del(os.environ["P4DIFF"])
 689             diff = p4_read_pipe("diff -du ...")
 690
 691             newdiff = ""
 692             for newFile in filesToAdd:
 693                 newdiff += "==== new file ====\n"
 694                 newdiff += "--- /dev/null\n"
 695                 newdiff += "+++ %s\n" % newFile
 696                 f = open(newFile, "r")
 697                 for line in f.readlines():
 698                     newdiff += "+" + line
 699                 f.close()
 700
 701             separatorLine = "######## everything below this line is just the diff #######\n"
 702
 703             [handle, fileName] = tempfile.mkstemp()
 704             tmpFile = os.fdopen(handle, "w+")
 705             if self.isWindows:
 706                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 707                 separatorLine = separatorLine.replace("\n", "\r\n")
 708                 newdiff = newdiff.replace("\n", "\r\n")
 709             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 710             tmpFile.close()
 711             defaultEditor = "vi"
 712             if platform.system() == "Windows":
 713                 defaultEditor = "notepad"
 714             if os.environ.has_key("P4EDITOR"):
 715                 editor = os.environ.get("P4EDITOR")
 716             else:
 717                 editor = os.environ.get("EDITOR", defaultEditor);
 718             system(editor + " " + fileName)
 719             tmpFile = open(fileName, "rb")
 720             message = tmpFile.read()
 721             tmpFile.close()
 722             os.remove(fileName)
 723             submitTemplate = message[:message.index(separatorLine)]
 724             if self.isWindows:
 725                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 726
 727             p4_write_pipe("submit -i", submitTemplate)
 728         else:
 729             fileName = "submit.txt"
 730             file = open(fileName, "w+")
 731             file.write(self.prepareLogMessage(template, logMessage))
 732             file.close()
 733             print ("Perforce submit template written as %s. "
 734                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 735                    % (fileName, fileName))
 736
 737     def run(self, args):
 738         if len(args) == 0:
 739             self.master = currentGitBranch()
 740             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 741                 die("Detecting current git branch failed!")
 742         elif len(args) == 1:
 743             self.master = args[0]
 744         else:
 745             return False
 746
 747         allowSubmit = gitConfig("git-p4.allowSubmit")
 748         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 749             die("%s is not in git-p4.allowSubmit" % self.master)
 750
 751         [upstream, settings] = findUpstreamBranchPoint()
 752         self.depotPath = settings['depot-paths'][0]
 753         if len(self.origin) == 0:
 754             self.origin = upstream
 755
 756         if self.verbose:
 757             print "Origin branch is " + self.origin
 758
 759         if len(self.depotPath) == 0:
 760             print "Internal error: cannot locate perforce depot path from existing branches"
 761             sys.exit(128)
 762
 763         self.clientPath = p4Where(self.depotPath)
 764
 765         if len(self.clientPath) == 0:
 766             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 767             sys.exit(128)
 768
 769         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 770         self.oldWorkingDirectory = os.getcwd()
 771
 772         chdir(self.clientPath)
 773         print "Syncronizing p4 checkout..."
 774         p4_system("sync ...")
 775
 776         self.check()
 777
 778         commits = []
 779         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 780             commits.append(line.strip())
 781         commits.reverse()
 782
 783         while len(commits) > 0:
 784             commit = commits[0]
 785             commits = commits[1:]
 786             self.applyCommit(commit)
 787             if not self.interactive:
 788                 break
 789
 790         if len(commits) == 0:
 791             print "All changes applied!"
 792             chdir(self.oldWorkingDirectory)
 793
 794             sync = P4Sync()
 795             sync.run([])
 796
 797             rebase = P4Rebase()
 798             rebase.rebase()
 799
 800         return True
 801
 802 class P4Sync(Command):
 803     def __init__(self):
 804         Command.__init__(self)
 805         self.options = [
 806                 optparse.make_option("--branch", dest="branch"),
 807                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 808                 optparse.make_option("--changesfile", dest="changesFile"),
 809                 optparse.make_option("--silent", dest="silent", action="store_true"),
 810                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 811                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 812                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 813                                      help="Import into refs/heads/ , not refs/remotes"),
 814                 optparse.make_option("--max-changes", dest="maxChanges"),
 815                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 816                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 817                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 818                                      help="Only sync files that are included in the Perforce Client Spec")
 819         ]
 820         self.description = """Imports from Perforce into a git repository.\n
 821     example:
 822     //depot/my/project/ -- to import the current head
 823     //depot/my/project/@all -- to import everything
 824     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 825
 826     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 827
 828         self.usage += " //depot/path[@revRange]"
 829         self.silent = False
 830         self.createdBranches = Set()
 831         self.committedChanges = Set()
 832         self.branch = ""
 833         self.detectBranches = False
 834         self.detectLabels = False
 835         self.changesFile = ""
 836         self.syncWithOrigin = True
 837         self.verbose = False
 838         self.importIntoRemotes = True
 839         self.maxChanges = ""
 840         self.isWindows = (platform.system() == "Windows")
 841         self.keepRepoPath = False
 842         self.depotPaths = None
 843         self.p4BranchesInGit = []
 844         self.cloneExclude = []
 845         self.useClientSpec = False
 846         self.clientSpecDirs = []
 847
 848         if gitConfig("git-p4.syncFromOrigin") == "false":
 849             self.syncWithOrigin = False
 850
 851     def extractFilesFromCommit(self, commit):
 852         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 853                              for path in self.cloneExclude]
 854         files = []
 855         fnum = 0
 856         while commit.has_key("depotFile%s" % fnum):
 857             path =  commit["depotFile%s" % fnum]
 858
 859             if [p for p in self.cloneExclude
 860                 if path.startswith (p)]:
 861                 found = False
 862             else:
 863                 found = [p for p in self.depotPaths
 864                          if path.startswith (p)]
 865             if not found:
 866                 fnum = fnum + 1
 867                 continue
 868
 869             file = {}
 870             file["path"] = path
 871             file["rev"] = commit["rev%s" % fnum]
 872             file["action"] = commit["action%s" % fnum]
 873             file["type"] = commit["type%s" % fnum]
 874             files.append(file)
 875             fnum = fnum + 1
 876         return files
 877
 878     def stripRepoPath(self, path, prefixes):
 879         if self.keepRepoPath:
 880             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 881
 882         for p in prefixes:
 883             if path.startswith(p):
 884                 path = path[len(p):]
 885
 886         return path
 887
 888     def splitFilesIntoBranches(self, commit):
 889         branches = {}
 890         fnum = 0
 891         while commit.has_key("depotFile%s" % fnum):
 892             path =  commit["depotFile%s" % fnum]
 893             found = [p for p in self.depotPaths
 894                      if path.startswith (p)]
 895             if not found:
 896                 fnum = fnum + 1
 897                 continue
 898
 899             file = {}
 900             file["path"] = path
 901             file["rev"] = commit["rev%s" % fnum]
 902             file["action"] = commit["action%s" % fnum]
 903             file["type"] = commit["type%s" % fnum]
 904             fnum = fnum + 1
 905
 906             relPath = self.stripRepoPath(path, self.depotPaths)
 907
 908             for branch in self.knownBranches.keys():
 909
 910                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 911                 if relPath.startswith(branch + "/"):
 912                     if branch not in branches:
 913                         branches[branch] = []
 914                     branches[branch].append(file)
 915                     break
 916
 917         return branches
 918
 919     ## Should move this out, doesn't use SELF.
 920     def readP4Files(self, files):
 921         filesForCommit = []
 922         filesToRead = []
 923
 924         for f in files:
 925             includeFile = True
 926             for val in self.clientSpecDirs:
 927                 if f['path'].startswith(val[0]):
 928                     if val[1] <= 0:
 929                         includeFile = False
 930                     break
 931
 932             if includeFile:
 933                 filesForCommit.append(f)
 934                 if f['action'] != 'delete':
 935                     filesToRead.append(f)
 936
 937         filedata = []
 938         if len(filesToRead) > 0:
 939             filedata = p4CmdList('-x - print',
 940                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 941                                                   for f in filesToRead]),
 942                                  stdin_mode='w+')
 943
 944             if "p4ExitCode" in filedata[0]:
 945                 die("Problems executing p4. Error: [%d]."
 946                     % (filedata[0]['p4ExitCode']));
 947
 948         j = 0;
 949         contents = {}
 950         while j < len(filedata):
 951             stat = filedata[j]
 952             j += 1
 953             text = [];
 954             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 955                 text.append(filedata[j]['data'])
 956                 j += 1
 957             text = ''.join(text)
 958
 959             if not stat.has_key('depotFile'):
 960                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 961                 continue
 962
 963             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 964                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 965             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 966                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 967
 968             contents[stat['depotFile']] = text
 969
 970         for f in filesForCommit:
 971             path = f['path']
 972             if contents.has_key(path):
 973                 f['data'] = contents[path]
 974
 975         return filesForCommit
 976
 977     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 978         epoch = details["time"]
 979         author = details["user"]
 980
 981         if self.verbose:
 982             print "commit into %s" % branch
 983
 984         # start with reading files; if that fails, we should not
 985         # create a commit.
 986         new_files = []
 987         for f in files:
 988             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 989                 new_files.append (f)
 990             else:
 991                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 992         files = self.readP4Files(new_files)
 993
 994         self.gitStream.write("commit %s\n" % branch)
 995 #        gitStream.write("mark :%s\n" % details["change"])
 996         self.committedChanges.add(int(details["change"]))
 997         committer = ""
 998         if author not in self.users:
 999             self.getUserMapFromPerforceServer()
1000         if author in self.users:
1001             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1002         else:
1003             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1004
1005         self.gitStream.write("committer %s\n" % committer)
1006
1007         self.gitStream.write("data <<EOT\n")
1008         self.gitStream.write(details["desc"])
1009         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1010                              % (','.join (branchPrefixes), details["change"]))
1011         if len(details['options']) > 0:
1012             self.gitStream.write(": options = %s" % details['options'])
1013         self.gitStream.write("]\nEOT\n\n")
1014
1015         if len(parent) > 0:
1016             if self.verbose:
1017                 print "parent %s" % parent
1018             self.gitStream.write("from %s\n" % parent)
1019
1020         for file in files:
1021             if file["type"] == "apple":
1022                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1023                 continue
1024
1025             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1026             if file["action"] == "delete":
1027                 self.gitStream.write("D %s\n" % relPath)
1028             else:
1029                 data = file['data']
1030
1031                 mode = "644"
1032                 if isP4Exec(file["type"]):
1033                     mode = "755"
1034                 elif file["type"] == "symlink":
1035                     mode = "120000"
1036                     # p4 print on a symlink contains "target\n", so strip it off
1037                     data = data[:-1]
1038
1039                 if self.isWindows and file["type"].endswith("text"):
1040                     data = data.replace("\r\n", "\n")
1041
1042                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1043                 self.gitStream.write("data %s\n" % len(data))
1044                 self.gitStream.write(data)
1045                 self.gitStream.write("\n")
1046
1047         self.gitStream.write("\n")
1048
1049         change = int(details["change"])
1050
1051         if self.labels.has_key(change):
1052             label = self.labels[change]
1053             labelDetails = label[0]
1054             labelRevisions = label[1]
1055             if self.verbose:
1056                 print "Change %s is labelled %s" % (change, labelDetails)
1057
1058             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1059                                                     for p in branchPrefixes]))
1060
1061             if len(files) == len(labelRevisions):
1062
1063                 cleanedFiles = {}
1064                 for info in files:
1065                     if info["action"] == "delete":
1066                         continue
1067                     cleanedFiles[info["depotFile"]] = info["rev"]
1068
1069                 if cleanedFiles == labelRevisions:
1070                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1071                     self.gitStream.write("from %s\n" % branch)
1072
1073                     owner = labelDetails["Owner"]
1074                     tagger = ""
1075                     if author in self.users:
1076                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1077                     else:
1078                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1079                     self.gitStream.write("tagger %s\n" % tagger)
1080                     self.gitStream.write("data <<EOT\n")
1081                     self.gitStream.write(labelDetails["Description"])
1082                     self.gitStream.write("EOT\n\n")
1083
1084                 else:
1085                     if not self.silent:
1086                         print ("Tag %s does not match with change %s: files do not match."
1087                                % (labelDetails["label"], change))
1088
1089             else:
1090                 if not self.silent:
1091                     print ("Tag %s does not match with change %s: file count is different."
1092                            % (labelDetails["label"], change))
1093
1094     def getUserCacheFilename(self):
1095         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1096         return home + "/.gitp4-usercache.txt"
1097
1098     def getUserMapFromPerforceServer(self):
1099         if self.userMapFromPerforceServer:
1100             return
1101         self.users = {}
1102
1103         for output in p4CmdList("users"):
1104             if not output.has_key("User"):
1105                 continue
1106             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1107
1108
1109         s = ''
1110         for (key, val) in self.users.items():
1111             s += "%s\t%s\n" % (key, val)
1112
1113         open(self.getUserCacheFilename(), "wb").write(s)
1114         self.userMapFromPerforceServer = True
1115
1116     def loadUserMapFromCache(self):
1117         self.users = {}
1118         self.userMapFromPerforceServer = False
1119         try:
1120             cache = open(self.getUserCacheFilename(), "rb")
1121             lines = cache.readlines()
1122             cache.close()
1123             for line in lines:
1124                 entry = line.strip().split("\t")
1125                 self.users[entry[0]] = entry[1]
1126         except IOError:
1127             self.getUserMapFromPerforceServer()
1128
1129     def getLabels(self):
1130         self.labels = {}
1131
1132         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1133         if len(l) > 0 and not self.silent:
1134             print "Finding files belonging to labels in %s" % `self.depotPaths`
1135
1136         for output in l:
1137             label = output["label"]
1138             revisions = {}
1139             newestChange = 0
1140             if self.verbose:
1141                 print "Querying files for label %s" % label
1142             for file in p4CmdList("files "
1143                                   +  ' '.join (["%s...@%s" % (p, label)
1144                                                 for p in self.depotPaths])):
1145                 revisions[file["depotFile"]] = file["rev"]
1146                 change = int(file["change"])
1147                 if change > newestChange:
1148                     newestChange = change
1149
1150             self.labels[newestChange] = [output, revisions]
1151
1152         if self.verbose:
1153             print "Label changes: %s" % self.labels.keys()
1154
1155     def guessProjectName(self):
1156         for p in self.depotPaths:
1157             if p.endswith("/"):
1158                 p = p[:-1]
1159             p = p[p.strip().rfind("/") + 1:]
1160             if not p.endswith("/"):
1161                p += "/"
1162             return p
1163
1164     def getBranchMapping(self):
1165         lostAndFoundBranches = set()
1166
1167         for info in p4CmdList("branches"):
1168             details = p4Cmd("branch -o %s" % info["branch"])
1169             viewIdx = 0
1170             while details.has_key("View%s" % viewIdx):
1171                 paths = details["View%s" % viewIdx].split(" ")
1172                 viewIdx = viewIdx + 1
1173                 # require standard //depot/foo/... //depot/bar/... mapping
1174                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1175                     continue
1176                 source = paths[0]
1177                 destination = paths[1]
1178                 ## HACK
1179                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1180                     source = source[len(self.depotPaths[0]):-4]
1181                     destination = destination[len(self.depotPaths[0]):-4]
1182
1183                     if destination in self.knownBranches:
1184                         if not self.silent:
1185                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1186                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1187                         continue
1188
1189                     self.knownBranches[destination] = source
1190
1191                     lostAndFoundBranches.discard(destination)
1192
1193                     if source not in self.knownBranches:
1194                         lostAndFoundBranches.add(source)
1195
1196
1197         for branch in lostAndFoundBranches:
1198             self.knownBranches[branch] = branch
1199
1200     def getBranchMappingFromGitBranches(self):
1201         branches = p4BranchesInGit(self.importIntoRemotes)
1202         for branch in branches.keys():
1203             if branch == "master":
1204                 branch = "main"
1205             else:
1206                 branch = branch[len(self.projectName):]
1207             self.knownBranches[branch] = branch
1208
1209     def listExistingP4GitBranches(self):
1210         # branches holds mapping from name to commit
1211         branches = p4BranchesInGit(self.importIntoRemotes)
1212         self.p4BranchesInGit = branches.keys()
1213         for branch in branches.keys():
1214             self.initialParents[self.refPrefix + branch] = branches[branch]
1215
1216     def updateOptionDict(self, d):
1217         option_keys = {}
1218         if self.keepRepoPath:
1219             option_keys['keepRepoPath'] = 1
1220
1221         d["options"] = ' '.join(sorted(option_keys.keys()))
1222
1223     def readOptions(self, d):
1224         self.keepRepoPath = (d.has_key('options')
1225                              and ('keepRepoPath' in d['options']))
1226
1227     def gitRefForBranch(self, branch):
1228         if branch == "main":
1229             return self.refPrefix + "master"
1230
1231         if len(branch) <= 0:
1232             return branch
1233
1234         return self.refPrefix + self.projectName + branch
1235
1236     def gitCommitByP4Change(self, ref, change):
1237         if self.verbose:
1238             print "looking in ref " + ref + " for change %s using bisect..." % change
1239
1240         earliestCommit = ""
1241         latestCommit = parseRevision(ref)
1242
1243         while True:
1244             if self.verbose:
1245                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1246             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1247             if len(next) == 0:
1248                 if self.verbose:
1249                     print "argh"
1250                 return ""
1251             log = extractLogMessageFromGitCommit(next)
1252             settings = extractSettingsGitLog(log)
1253             currentChange = int(settings['change'])
1254             if self.verbose:
1255                 print "current change %s" % currentChange
1256
1257             if currentChange == change:
1258                 if self.verbose:
1259                     print "found %s" % next
1260                 return next
1261
1262             if currentChange < change:
1263                 earliestCommit = "^%s" % next
1264             else:
1265                 latestCommit = "%s" % next
1266
1267         return ""
1268
1269     def importNewBranch(self, branch, maxChange):
1270         # make fast-import flush all changes to disk and update the refs using the checkpoint
1271         # command so that we can try to find the branch parent in the git history
1272         self.gitStream.write("checkpoint\n\n");
1273         self.gitStream.flush();
1274         branchPrefix = self.depotPaths[0] + branch + "/"
1275         range = "@1,%s" % maxChange
1276         #print "prefix" + branchPrefix
1277         changes = p4ChangesForPaths([branchPrefix], range)
1278         if len(changes) <= 0:
1279             return False
1280         firstChange = changes[0]
1281         #print "first change in branch: %s" % firstChange
1282         sourceBranch = self.knownBranches[branch]
1283         sourceDepotPath = self.depotPaths[0] + sourceBranch
1284         sourceRef = self.gitRefForBranch(sourceBranch)
1285         #print "source " + sourceBranch
1286
1287         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1288         #print "branch parent: %s" % branchParentChange
1289         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1290         if len(gitParent) > 0:
1291             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1292             #print "parent git commit: %s" % gitParent
1293
1294         self.importChanges(changes)
1295         return True
1296
1297     def importChanges(self, changes):
1298         cnt = 1
1299         for change in changes:
1300             description = p4Cmd("describe %s" % change)
1301             self.updateOptionDict(description)
1302
1303             if not self.silent:
1304                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1305                 sys.stdout.flush()
1306             cnt = cnt + 1
1307
1308             try:
1309                 if self.detectBranches:
1310                     branches = self.splitFilesIntoBranches(description)
1311                     for branch in branches.keys():
1312                         ## HACK  --hwn
1313                         branchPrefix = self.depotPaths[0] + branch + "/"
1314
1315                         parent = ""
1316
1317                         filesForCommit = branches[branch]
1318
1319                         if self.verbose:
1320                             print "branch is %s" % branch
1321
1322                         self.updatedBranches.add(branch)
1323
1324                         if branch not in self.createdBranches:
1325                             self.createdBranches.add(branch)
1326                             parent = self.knownBranches[branch]
1327                             if parent == branch:
1328                                 parent = ""
1329                             else:
1330                                 fullBranch = self.projectName + branch
1331                                 if fullBranch not in self.p4BranchesInGit:
1332                                     if not self.silent:
1333                                         print("\n    Importing new branch %s" % fullBranch);
1334                                     if self.importNewBranch(branch, change - 1):
1335                                         parent = ""
1336                                         self.p4BranchesInGit.append(fullBranch)
1337                                     if not self.silent:
1338                                         print("\n    Resuming with change %s" % change);
1339
1340                                 if self.verbose:
1341                                     print "parent determined through known branches: %s" % parent
1342
1343                         branch = self.gitRefForBranch(branch)
1344                         parent = self.gitRefForBranch(parent)
1345
1346                         if self.verbose:
1347                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1348
1349                         if len(parent) == 0 and branch in self.initialParents:
1350                             parent = self.initialParents[branch]
1351                             del self.initialParents[branch]
1352
1353                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1354                 else:
1355                     files = self.extractFilesFromCommit(description)
1356                     self.commit(description, files, self.branch, self.depotPaths,
1357                                 self.initialParent)
1358                     self.initialParent = ""
1359             except IOError:
1360                 print self.gitError.read()
1361                 sys.exit(1)
1362
1363     def importHeadRevision(self, revision):
1364         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1365
1366         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1367         details["desc"] = ("Initial import of %s from the state at revision %s"
1368                            % (' '.join(self.depotPaths), revision))
1369         details["change"] = revision
1370         newestRevision = 0
1371
1372         fileCnt = 0
1373         for info in p4CmdList("files "
1374                               +  ' '.join(["%s...%s"
1375                                            % (p, revision)
1376                                            for p in self.depotPaths])):
1377
1378             if info['code'] == 'error':
1379                 sys.stderr.write("p4 returned an error: %s\n"
1380                                  % info['data'])
1381                 sys.exit(1)
1382
1383
1384             change = int(info["change"])
1385             if change > newestRevision:
1386                 newestRevision = change
1387
1388             if info["action"] == "delete":
1389                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1390                 #fileCnt = fileCnt + 1
1391                 continue
1392
1393             for prop in ["depotFile", "rev", "action", "type" ]:
1394                 details["%s%s" % (prop, fileCnt)] = info[prop]
1395
1396             fileCnt = fileCnt + 1
1397
1398         details["change"] = newestRevision
1399         self.updateOptionDict(details)
1400         try:
1401             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1402         except IOError:
1403             print "IO error with git fast-import. Is your git version recent enough?"
1404             print self.gitError.read()
1405
1406
1407     def getClientSpec(self):
1408         specList = p4CmdList( "client -o" )
1409         temp = {}
1410         for entry in specList:
1411             for k,v in entry.iteritems():
1412                 if k.startswith("View"):
1413                     if v.startswith('"'):
1414                         start = 1
1415                     else:
1416                         start = 0
1417                     index = v.find("...")
1418                     v = v[start:index]
1419                     if v.startswith("-"):
1420                         v = v[1:]
1421                         temp[v] = -len(v)
1422                     else:
1423                         temp[v] = len(v)
1424         self.clientSpecDirs = temp.items()
1425         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1426
1427     def run(self, args):
1428         self.depotPaths = []
1429         self.changeRange = ""
1430         self.initialParent = ""
1431         self.previousDepotPaths = []
1432
1433         # map from branch depot path to parent branch
1434         self.knownBranches = {}
1435         self.initialParents = {}
1436         self.hasOrigin = originP4BranchesExist()
1437         if not self.syncWithOrigin:
1438             self.hasOrigin = False
1439
1440         if self.importIntoRemotes:
1441             self.refPrefix = "refs/remotes/p4/"
1442         else:
1443             self.refPrefix = "refs/heads/p4/"
1444
1445         if self.syncWithOrigin and self.hasOrigin:
1446             if not self.silent:
1447                 print "Syncing with origin first by calling git fetch origin"
1448             system("git fetch origin")
1449
1450         if len(self.branch) == 0:
1451             self.branch = self.refPrefix + "master"
1452             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1453                 system("git update-ref %s refs/heads/p4" % self.branch)
1454                 system("git branch -D p4");
1455             # create it /after/ importing, when master exists
1456             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1457                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1458
1459         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1460             self.getClientSpec()
1461
1462         # TODO: should always look at previous commits,
1463         # merge with previous imports, if possible.
1464         if args == []:
1465             if self.hasOrigin:
1466                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1467             self.listExistingP4GitBranches()
1468
1469             if len(self.p4BranchesInGit) > 1:
1470                 if not self.silent:
1471                     print "Importing from/into multiple branches"
1472                 self.detectBranches = True
1473
1474             if self.verbose:
1475                 print "branches: %s" % self.p4BranchesInGit
1476
1477             p4Change = 0
1478             for branch in self.p4BranchesInGit:
1479                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1480
1481                 settings = extractSettingsGitLog(logMsg)
1482
1483                 self.readOptions(settings)
1484                 if (settings.has_key('depot-paths')
1485                     and settings.has_key ('change')):
1486                     change = int(settings['change']) + 1
1487                     p4Change = max(p4Change, change)
1488
1489                     depotPaths = sorted(settings['depot-paths'])
1490                     if self.previousDepotPaths == []:
1491                         self.previousDepotPaths = depotPaths
1492                     else:
1493                         paths = []
1494                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1495                             for i in range(0, min(len(cur), len(prev))):
1496                                 if cur[i] <> prev[i]:
1497                                     i = i - 1
1498                                     break
1499
1500                             paths.append (cur[:i + 1])
1501
1502                         self.previousDepotPaths = paths
1503
1504             if p4Change > 0:
1505                 self.depotPaths = sorted(self.previousDepotPaths)
1506                 self.changeRange = "@%s,#head" % p4Change
1507                 if not self.detectBranches:
1508                     self.initialParent = parseRevision(self.branch)
1509                 if not self.silent and not self.detectBranches:
1510                     print "Performing incremental import into %s git branch" % self.branch
1511
1512         if not self.branch.startswith("refs/"):
1513             self.branch = "refs/heads/" + self.branch
1514
1515         if len(args) == 0 and self.depotPaths:
1516             if not self.silent:
1517                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1518         else:
1519             if self.depotPaths and self.depotPaths != args:
1520                 print ("previous import used depot path %s and now %s was specified. "
1521                        "This doesn't work!" % (' '.join (self.depotPaths),
1522                                                ' '.join (args)))
1523                 sys.exit(1)
1524
1525             self.depotPaths = sorted(args)
1526
1527         revision = ""
1528         self.users = {}
1529
1530         newPaths = []
1531         for p in self.depotPaths:
1532             if p.find("@") != -1:
1533                 atIdx = p.index("@")
1534                 self.changeRange = p[atIdx:]
1535                 if self.changeRange == "@all":
1536                     self.changeRange = ""
1537                 elif ',' not in self.changeRange:
1538                     revision = self.changeRange
1539                     self.changeRange = ""
1540                 p = p[:atIdx]
1541             elif p.find("#") != -1:
1542                 hashIdx = p.index("#")
1543                 revision = p[hashIdx:]
1544                 p = p[:hashIdx]
1545             elif self.previousDepotPaths == []:
1546                 revision = "#head"
1547
1548             p = re.sub ("\.\.\.$", "", p)
1549             if not p.endswith("/"):
1550                 p += "/"
1551
1552             newPaths.append(p)
1553
1554         self.depotPaths = newPaths
1555
1556
1557         self.loadUserMapFromCache()
1558         self.labels = {}
1559         if self.detectLabels:
1560             self.getLabels();
1561
1562         if self.detectBranches:
1563             ## FIXME - what's a P4 projectName ?
1564             self.projectName = self.guessProjectName()
1565
1566             if self.hasOrigin:
1567                 self.getBranchMappingFromGitBranches()
1568             else:
1569                 self.getBranchMapping()
1570             if self.verbose:
1571                 print "p4-git branches: %s" % self.p4BranchesInGit
1572                 print "initial parents: %s" % self.initialParents
1573             for b in self.p4BranchesInGit:
1574                 if b != "master":
1575
1576                     ## FIXME
1577                     b = b[len(self.projectName):]
1578                 self.createdBranches.add(b)
1579
1580         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1581
1582         importProcess = subprocess.Popen(["git", "fast-import"],
1583                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1584                                          stderr=subprocess.PIPE);
1585         self.gitOutput = importProcess.stdout
1586         self.gitStream = importProcess.stdin
1587         self.gitError = importProcess.stderr
1588
1589         if revision:
1590             self.importHeadRevision(revision)
1591         else:
1592             changes = []
1593
1594             if len(self.changesFile) > 0:
1595                 output = open(self.changesFile).readlines()
1596                 changeSet = Set()
1597                 for line in output:
1598                     changeSet.add(int(line))
1599
1600                 for change in changeSet:
1601                     changes.append(change)
1602
1603                 changes.sort()
1604             else:
1605                 if self.verbose:
1606                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1607                                                               self.changeRange)
1608                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1609
1610                 if len(self.maxChanges) > 0:
1611                     changes = changes[:min(int(self.maxChanges), len(changes))]
1612
1613             if len(changes) == 0:
1614                 if not self.silent:
1615                     print "No changes to import!"
1616                 return True
1617
1618             if not self.silent and not self.detectBranches:
1619                 print "Import destination: %s" % self.branch
1620
1621             self.updatedBranches = set()
1622
1623             self.importChanges(changes)
1624
1625             if not self.silent:
1626                 print ""
1627                 if len(self.updatedBranches) > 0:
1628                     sys.stdout.write("Updated branches: ")
1629                     for b in self.updatedBranches:
1630                         sys.stdout.write("%s " % b)
1631                     sys.stdout.write("\n")
1632
1633         self.gitStream.close()
1634         if importProcess.wait() != 0:
1635             die("fast-import failed: %s" % self.gitError.read())
1636         self.gitOutput.close()
1637         self.gitError.close()
1638
1639         return True
1640
1641 class P4Rebase(Command):
1642     def __init__(self):
1643         Command.__init__(self)
1644         self.options = [ ]
1645         self.description = ("Fetches the latest revision from perforce and "
1646                             + "rebases the current work (branch) against it")
1647         self.verbose = False
1648
1649     def run(self, args):
1650         sync = P4Sync()
1651         sync.run([])
1652
1653         return self.rebase()
1654
1655     def rebase(self):
1656         if os.system("git update-index --refresh") != 0:
1657             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1658         if len(read_pipe("git diff-index HEAD --")) > 0:
1659             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1660
1661         [upstream, settings] = findUpstreamBranchPoint()
1662         if len(upstream) == 0:
1663             die("Cannot find upstream branchpoint for rebase")
1664
1665         # the branchpoint may be p4/foo~3, so strip off the parent
1666         upstream = re.sub("~[0-9]+$", "", upstream)
1667
1668         print "Rebasing the current branch onto %s" % upstream
1669         oldHead = read_pipe("git rev-parse HEAD").strip()
1670         system("git rebase %s" % upstream)
1671         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1672         return True
1673
1674 class P4Clone(P4Sync):
1675     def __init__(self):
1676         P4Sync.__init__(self)
1677         self.description = "Creates a new git repository and imports from Perforce into it"
1678         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1679         self.options += [
1680             optparse.make_option("--destination", dest="cloneDestination",
1681                                  action='store', default=None,
1682                                  help="where to leave result of the clone"),
1683             optparse.make_option("-/", dest="cloneExclude",
1684                                  action="append", type="string",
1685                                  help="exclude depot path")
1686         ]
1687         self.cloneDestination = None
1688         self.needsGit = False
1689
1690     # This is required for the "append" cloneExclude action
1691     def ensure_value(self, attr, value):
1692         if not hasattr(self, attr) or getattr(self, attr) is None:
1693             setattr(self, attr, value)
1694         return getattr(self, attr)
1695
1696     def defaultDestination(self, args):
1697         ## TODO: use common prefix of args?
1698         depotPath = args[0]
1699         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1700         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1701         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1702         depotDir = re.sub(r"/$", "", depotDir)
1703         return os.path.split(depotDir)[1]
1704
1705     def run(self, args):
1706         if len(args) < 1:
1707             return False
1708
1709         if self.keepRepoPath and not self.cloneDestination:
1710             sys.stderr.write("Must specify destination for --keep-path\n")
1711             sys.exit(1)
1712
1713         depotPaths = args
1714
1715         if not self.cloneDestination and len(depotPaths) > 1:
1716             self.cloneDestination = depotPaths[-1]
1717             depotPaths = depotPaths[:-1]
1718
1719         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1720         for p in depotPaths:
1721             if not p.startswith("//"):
1722                 return False
1723
1724         if not self.cloneDestination:
1725             self.cloneDestination = self.defaultDestination(args)
1726
1727         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1728         if not os.path.exists(self.cloneDestination):
1729             os.makedirs(self.cloneDestination)
1730         chdir(self.cloneDestination)
1731         system("git init")
1732         self.gitdir = os.getcwd() + "/.git"
1733         if not P4Sync.run(self, depotPaths):
1734             return False
1735         if self.branch != "master":
1736             if gitBranchExists("refs/remotes/p4/master"):
1737                 system("git branch master refs/remotes/p4/master")
1738                 system("git checkout -f")
1739             else:
1740                 print "Could not detect main branch. No checkout/master branch created."
1741
1742         return True
1743
1744 class P4Branches(Command):
1745     def __init__(self):
1746         Command.__init__(self)
1747         self.options = [ ]
1748         self.description = ("Shows the git branches that hold imports and their "
1749                             + "corresponding perforce depot paths")
1750         self.verbose = False
1751
1752     def run(self, args):
1753         if originP4BranchesExist():
1754             createOrUpdateBranchesFromOrigin()
1755
1756         cmdline = "git rev-parse --symbolic "
1757         cmdline += " --remotes"
1758
1759         for line in read_pipe_lines(cmdline):
1760             line = line.strip()
1761
1762             if not line.startswith('p4/') or line == "p4/HEAD":
1763                 continue
1764             branch = line
1765
1766             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1767             settings = extractSettingsGitLog(log)
1768
1769             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1770         return True
1771
1772 class HelpFormatter(optparse.IndentedHelpFormatter):
1773     def __init__(self):
1774         optparse.IndentedHelpFormatter.__init__(self)
1775
1776     def format_description(self, description):
1777         if description:
1778             return description + "\n"
1779         else:
1780             return ""
1781
1782 def printUsage(commands):
1783     print "usage: %s <command> [options]" % sys.argv[0]
1784     print ""
1785     print "valid commands: %s" % ", ".join(commands)
1786     print ""
1787     print "Try %s <command> --help for command specific help." % sys.argv[0]
1788     print ""
1789
1790 commands = {
1791     "debug" : P4Debug,
1792     "submit" : P4Submit,
1793     "commit" : P4Submit,
1794     "sync" : P4Sync,
1795     "rebase" : P4Rebase,
1796     "clone" : P4Clone,
1797     "rollback" : P4RollBack,
1798     "branches" : P4Branches
1799 }
1800
1801
1802 def main():
1803     if len(sys.argv[1:]) == 0:
1804         printUsage(commands.keys())
1805         sys.exit(2)
1806
1807     cmd = ""
1808     cmdName = sys.argv[1]
1809     try:
1810         klass = commands[cmdName]
1811         cmd = klass()
1812     except KeyError:
1813         print "unknown command %s" % cmdName
1814         print ""
1815         printUsage(commands.keys())
1816         sys.exit(2)
1817
1818     options = cmd.options
1819     cmd.gitdir = os.environ.get("GIT_DIR", None)
1820
1821     args = sys.argv[2:]
1822
1823     if len(options) > 0:
1824         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1825
1826         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1827                                        options,
1828                                        description = cmd.description,
1829                                        formatter = HelpFormatter())
1830
1831         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1832     global verbose
1833     verbose = cmd.verbose
1834     if cmd.needsGit:
1835         if cmd.gitdir == None:
1836             cmd.gitdir = os.path.abspath(".git")
1837             if not isValidGitDir(cmd.gitdir):
1838                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1839                 if os.path.exists(cmd.gitdir):
1840                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1841                     if len(cdup) > 0:
1842                         chdir(cdup);
1843
1844         if not isValidGitDir(cmd.gitdir):
1845             if isValidGitDir(cmd.gitdir + "/.git"):
1846                 cmd.gitdir += "/.git"
1847             else:
1848                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1849
1850         os.environ["GIT_DIR"] = cmd.gitdir
1851
1852     if not cmd.run(args):
1853         parser.print_help()
1854
1855
1856 if __name__ == '__main__':
1857     main()