contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     if os.name == 'nt':
  56         os.environ['PWD']=dir
  57     os.chdir(dir)
  58
  59 def die(msg):
  60     if verbose:
  61         raise Exception(msg)
  62     else:
  63         sys.stderr.write(msg + "\n")
  64         sys.exit(1)
  65
  66 def write_pipe(c, str):
  67     if verbose:
  68         sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70     pipe = os.popen(c, 'w')
  71     val = pipe.write(str)
  72     if pipe.close():
  73         die('Command failed: %s' % c)
  74
  75     return val
  76
  77 def p4_write_pipe(c, str):
  78     real_cmd = p4_build_cmd(c)
  79     return write_pipe(real_cmd, str)
  80
  81 def read_pipe(c, ignore_error=False):
  82     if verbose:
  83         sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85     pipe = os.popen(c, 'rb')
  86     val = pipe.read()
  87     if pipe.close() and not ignore_error:
  88         die('Command failed: %s' % c)
  89
  90     return val
  91
  92 def p4_read_pipe(c, ignore_error=False):
  93     real_cmd = p4_build_cmd(c)
  94     return read_pipe(real_cmd, ignore_error)
  95
  96 def read_pipe_lines(c):
  97     if verbose:
  98         sys.stderr.write('Reading pipe: %s\n' % c)
  99     ## todo: check return status
 100     pipe = os.popen(c, 'rb')
 101     val = pipe.readlines()
 102     if pipe.close():
 103         die('Command failed: %s' % c)
 104
 105     return val
 106
 107 def p4_read_pipe_lines(c):
 108     """Specifically invoke p4 on the command supplied. """
 109     real_cmd = p4_build_cmd(c)
 110     return read_pipe_lines(real_cmd)
 111
 112 def system(cmd):
 113     if verbose:
 114         sys.stderr.write("executing %s\n" % cmd)
 115     if os.system(cmd) != 0:
 116         die("command failed: %s" % cmd)
 117
 118 def p4_system(cmd):
 119     """Specifically invoke p4 as the system command. """
 120     real_cmd = p4_build_cmd(cmd)
 121     return system(real_cmd)
 122
 123 def isP4Exec(kind):
 124     """Determine if a Perforce 'kind' should have execute permission
 125
 126     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128     a plus sign, it is also executable"""
 129     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131 def setP4ExecBit(file, mode):
 132     # Reopens an already open file and changes the execute bit to match
 133     # the execute bit setting in the passed in mode.
 134
 135     p4Type = "+x"
 136
 137     if not isModeExec(mode):
 138         p4Type = getP4OpenedType(file)
 139         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141         if p4Type[-1] == "+":
 142             p4Type = p4Type[0:-1]
 143
 144     p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146 def getP4OpenedType(file):
 147     # Returns the perforce file type for the given file.
 148
 149     result = p4_read_pipe("opened %s" % file)
 150     match = re.match(".*\((.+)\)\r?$", result)
 151     if match:
 152         return match.group(1)
 153     else:
 154         die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156 def diffTreePattern():
 157     # This is a simple generator for the diff tree regex pattern. This could be
 158     # a class variable if this and parseDiffTreeEntry were a part of a class.
 159     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160     while True:
 161         yield pattern
 162
 163 def parseDiffTreeEntry(entry):
 164     """Parses a single diff tree entry into its component elements.
 165
 166     See git-diff-tree(1) manpage for details about the format of the diff
 167     output. This method returns a dictionary with the following elements:
 168
 169     src_mode - The mode of the source file
 170     dst_mode - The mode of the destination file
 171     src_sha1 - The sha1 for the source file
 172     dst_sha1 - The sha1 fr the destination file
 173     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174     status_score - The score for the status (applicable for 'C' and 'R'
 175                    statuses). This is None if there is no score.
 176     src - The path for the source file.
 177     dst - The path for the destination file. This is only present for
 178           copy or renames. If it is not present, this is None.
 179
 180     If the pattern is not matched, None is returned."""
 181
 182     match = diffTreePattern().next().match(entry)
 183     if match:
 184         return {
 185             'src_mode': match.group(1),
 186             'dst_mode': match.group(2),
 187             'src_sha1': match.group(3),
 188             'dst_sha1': match.group(4),
 189             'status': match.group(5),
 190             'status_score': match.group(6),
 191             'src': match.group(7),
 192             'dst': match.group(10)
 193         }
 194     return None
 195
 196 def isModeExec(mode):
 197     # Returns True if the given git mode represents an executable file,
 198     # otherwise False.
 199     return mode[-3:] == "755"
 200
 201 def isModeExecChanged(src_mode, dst_mode):
 202     return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205     cmd = p4_build_cmd("-G %s" % (cmd))
 206     if verbose:
 207         sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209     # Use a temporary file to avoid deadlocks without
 210     # subprocess.communicate(), which would put another copy
 211     # of stdout into memory.
 212     stdin_file = None
 213     if stdin is not None:
 214         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215         stdin_file.write(stdin)
 216         stdin_file.flush()
 217         stdin_file.seek(0)
 218
 219     p4 = subprocess.Popen(cmd, shell=True,
 220                           stdin=stdin_file,
 221                           stdout=subprocess.PIPE)
 222
 223     result = []
 224     try:
 225         while True:
 226             entry = marshal.load(p4.stdout)
 227             result.append(entry)
 228     except EOFError:
 229         pass
 230     exitCode = p4.wait()
 231     if exitCode != 0:
 232         entry = {}
 233         entry["p4ExitCode"] = exitCode
 234         result.append(entry)
 235
 236     return result
 237
 238 def p4Cmd(cmd):
 239     list = p4CmdList(cmd)
 240     result = {}
 241     for entry in list:
 242         result.update(entry)
 243     return result;
 244
 245 def p4Where(depotPath):
 246     if not depotPath.endswith("/"):
 247         depotPath += "/"
 248     output = p4Cmd("where %s..." % depotPath)
 249     if output["code"] == "error":
 250         return ""
 251     clientPath = ""
 252     if "path" in output:
 253         clientPath = output.get("path")
 254     elif "data" in output:
 255         data = output.get("data")
 256         lastSpace = data.rfind(" ")
 257         clientPath = data[lastSpace + 1:]
 258
 259     if clientPath.endswith("..."):
 260         clientPath = clientPath[:-3]
 261     return clientPath
 262
 263 def currentGitBranch():
 264     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 265
 266 def isValidGitDir(path):
 267     if (os.path.exists(path + "/HEAD")
 268         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 269         return True;
 270     return False
 271
 272 def parseRevision(ref):
 273     return read_pipe("git rev-parse %s" % ref).strip()
 274
 275 def extractLogMessageFromGitCommit(commit):
 276     logMessage = ""
 277
 278     ## fixme: title is first line of commit, not 1st paragraph.
 279     foundTitle = False
 280     for log in read_pipe_lines("git cat-file commit %s" % commit):
 281        if not foundTitle:
 282            if len(log) == 1:
 283                foundTitle = True
 284            continue
 285
 286        logMessage += log
 287     return logMessage
 288
 289 def extractSettingsGitLog(log):
 290     values = {}
 291     for line in log.split("\n"):
 292         line = line.strip()
 293         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 294         if not m:
 295             continue
 296
 297         assignments = m.group(1).split (':')
 298         for a in assignments:
 299             vals = a.split ('=')
 300             key = vals[0].strip()
 301             val = ('='.join (vals[1:])).strip()
 302             if val.endswith ('\"') and val.startswith('"'):
 303                 val = val[1:-1]
 304
 305             values[key] = val
 306
 307     paths = values.get("depot-paths")
 308     if not paths:
 309         paths = values.get("depot-path")
 310     if paths:
 311         values['depot-paths'] = paths.split(',')
 312     return values
 313
 314 def gitBranchExists(branch):
 315     proc = subprocess.Popen(["git", "rev-parse", branch],
 316                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 317     return proc.wait() == 0;
 318
 319 def gitConfig(key):
 320     return read_pipe("git config %s" % key, ignore_error=True).strip()
 321
 322 def p4BranchesInGit(branchesAreInRemotes = True):
 323     branches = {}
 324
 325     cmdline = "git rev-parse --symbolic "
 326     if branchesAreInRemotes:
 327         cmdline += " --remotes"
 328     else:
 329         cmdline += " --branches"
 330
 331     for line in read_pipe_lines(cmdline):
 332         line = line.strip()
 333
 334         ## only import to p4/
 335         if not line.startswith('p4/') or line == "p4/HEAD":
 336             continue
 337         branch = line
 338
 339         # strip off p4
 340         branch = re.sub ("^p4/", "", line)
 341
 342         branches[branch] = parseRevision(line)
 343     return branches
 344
 345 def findUpstreamBranchPoint(head = "HEAD"):
 346     branches = p4BranchesInGit()
 347     # map from depot-path to branch name
 348     branchByDepotPath = {}
 349     for branch in branches.keys():
 350         tip = branches[branch]
 351         log = extractLogMessageFromGitCommit(tip)
 352         settings = extractSettingsGitLog(log)
 353         if settings.has_key("depot-paths"):
 354             paths = ",".join(settings["depot-paths"])
 355             branchByDepotPath[paths] = "remotes/p4/" + branch
 356
 357     settings = None
 358     parent = 0
 359     while parent < 65535:
 360         commit = head + "~%s" % parent
 361         log = extractLogMessageFromGitCommit(commit)
 362         settings = extractSettingsGitLog(log)
 363         if settings.has_key("depot-paths"):
 364             paths = ",".join(settings["depot-paths"])
 365             if branchByDepotPath.has_key(paths):
 366                 return [branchByDepotPath[paths], settings]
 367
 368         parent = parent + 1
 369
 370     return ["", settings]
 371
 372 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 373     if not silent:
 374         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 375                % localRefPrefix)
 376
 377     originPrefix = "origin/p4/"
 378
 379     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 380         line = line.strip()
 381         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 382             continue
 383
 384         headName = line[len(originPrefix):]
 385         remoteHead = localRefPrefix + headName
 386         originHead = line
 387
 388         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 389         if (not original.has_key('depot-paths')
 390             or not original.has_key('change')):
 391             continue
 392
 393         update = False
 394         if not gitBranchExists(remoteHead):
 395             if verbose:
 396                 print "creating %s" % remoteHead
 397             update = True
 398         else:
 399             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 400             if settings.has_key('change') > 0:
 401                 if settings['depot-paths'] == original['depot-paths']:
 402                     originP4Change = int(original['change'])
 403                     p4Change = int(settings['change'])
 404                     if originP4Change > p4Change:
 405                         print ("%s (%s) is newer than %s (%s). "
 406                                "Updating p4 branch from origin."
 407                                % (originHead, originP4Change,
 408                                   remoteHead, p4Change))
 409                         update = True
 410                 else:
 411                     print ("Ignoring: %s was imported from %s while "
 412                            "%s was imported from %s"
 413                            % (originHead, ','.join(original['depot-paths']),
 414                               remoteHead, ','.join(settings['depot-paths'])))
 415
 416         if update:
 417             system("git update-ref %s %s" % (remoteHead, originHead))
 418
 419 def originP4BranchesExist():
 420         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 421
 422 def p4ChangesForPaths(depotPaths, changeRange):
 423     assert depotPaths
 424     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 425                                                         for p in depotPaths]))
 426
 427     changes = []
 428     for line in output:
 429         changeNum = line.split(" ")[1]
 430         changes.append(int(changeNum))
 431
 432     changes.sort()
 433     return changes
 434
 435 class Command:
 436     def __init__(self):
 437         self.usage = "usage: %prog [options]"
 438         self.needsGit = True
 439
 440 class P4Debug(Command):
 441     def __init__(self):
 442         Command.__init__(self)
 443         self.options = [
 444             optparse.make_option("--verbose", dest="verbose", action="store_true",
 445                                  default=False),
 446             ]
 447         self.description = "A tool to debug the output of p4 -G."
 448         self.needsGit = False
 449         self.verbose = False
 450
 451     def run(self, args):
 452         j = 0
 453         for output in p4CmdList(" ".join(args)):
 454             print 'Element: %d' % j
 455             j += 1
 456             print output
 457         return True
 458
 459 class P4RollBack(Command):
 460     def __init__(self):
 461         Command.__init__(self)
 462         self.options = [
 463             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 464             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 465         ]
 466         self.description = "A tool to debug the multi-branch import. Don't use :)"
 467         self.verbose = False
 468         self.rollbackLocalBranches = False
 469
 470     def run(self, args):
 471         if len(args) != 1:
 472             return False
 473         maxChange = int(args[0])
 474
 475         if "p4ExitCode" in p4Cmd("changes -m 1"):
 476             die("Problems executing p4");
 477
 478         if self.rollbackLocalBranches:
 479             refPrefix = "refs/heads/"
 480             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 481         else:
 482             refPrefix = "refs/remotes/"
 483             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 484
 485         for line in lines:
 486             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 487                 line = line.strip()
 488                 ref = refPrefix + line
 489                 log = extractLogMessageFromGitCommit(ref)
 490                 settings = extractSettingsGitLog(log)
 491
 492                 depotPaths = settings['depot-paths']
 493                 change = settings['change']
 494
 495                 changed = False
 496
 497                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 498                                                            for p in depotPaths]))) == 0:
 499                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 500                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 501                     continue
 502
 503                 while change and int(change) > maxChange:
 504                     changed = True
 505                     if self.verbose:
 506                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 507                     system("git update-ref %s \"%s^\"" % (ref, ref))
 508                     log = extractLogMessageFromGitCommit(ref)
 509                     settings =  extractSettingsGitLog(log)
 510
 511
 512                     depotPaths = settings['depot-paths']
 513                     change = settings['change']
 514
 515                 if changed:
 516                     print "%s rewound to %s" % (ref, change)
 517
 518         return True
 519
 520 class P4Submit(Command):
 521     def __init__(self):
 522         Command.__init__(self)
 523         self.options = [
 524                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 525                 optparse.make_option("--origin", dest="origin"),
 526                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 527         ]
 528         self.description = "Submit changes from git to the perforce depot."
 529         self.usage += " [name of git branch to submit into perforce depot]"
 530         self.interactive = True
 531         self.origin = ""
 532         self.detectRename = False
 533         self.verbose = False
 534         self.isWindows = (platform.system() == "Windows")
 535
 536     def check(self):
 537         if len(p4CmdList("opened ...")) > 0:
 538             die("You have files opened with perforce! Close them before starting the sync.")
 539
 540     # replaces everything between 'Description:' and the next P4 submit template field with the
 541     # commit message
 542     def prepareLogMessage(self, template, message):
 543         result = ""
 544
 545         inDescriptionSection = False
 546
 547         for line in template.split("\n"):
 548             if line.startswith("#"):
 549                 result += line + "\n"
 550                 continue
 551
 552             if inDescriptionSection:
 553                 if line.startswith("Files:"):
 554                     inDescriptionSection = False
 555                 else:
 556                     continue
 557             else:
 558                 if line.startswith("Description:"):
 559                     inDescriptionSection = True
 560                     line += "\n"
 561                     for messageLine in message.split("\n"):
 562                         line += "\t" + messageLine + "\n"
 563
 564             result += line + "\n"
 565
 566         return result
 567
 568     def prepareSubmitTemplate(self):
 569         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 570         template = ""
 571         inFilesSection = False
 572         for line in p4_read_pipe_lines("change -o"):
 573             if line.endswith("\r\n"):
 574                 line = line[:-2] + "\n"
 575             if inFilesSection:
 576                 if line.startswith("\t"):
 577                     # path starts and ends with a tab
 578                     path = line[1:]
 579                     lastTab = path.rfind("\t")
 580                     if lastTab != -1:
 581                         path = path[:lastTab]
 582                         if not path.startswith(self.depotPath):
 583                             continue
 584                 else:
 585                     inFilesSection = False
 586             else:
 587                 if line.startswith("Files:"):
 588                     inFilesSection = True
 589
 590             template += line
 591
 592         return template
 593
 594     def applyCommit(self, id):
 595         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 596         diffOpts = ("", "-M")[self.detectRename]
 597         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 598         filesToAdd = set()
 599         filesToDelete = set()
 600         editedFiles = set()
 601         filesToChangeExecBit = {}
 602         for line in diff:
 603             diff = parseDiffTreeEntry(line)
 604             modifier = diff['status']
 605             path = diff['src']
 606             if modifier == "M":
 607                 p4_system("edit \"%s\"" % path)
 608                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 609                     filesToChangeExecBit[path] = diff['dst_mode']
 610                 editedFiles.add(path)
 611             elif modifier == "A":
 612                 filesToAdd.add(path)
 613                 filesToChangeExecBit[path] = diff['dst_mode']
 614                 if path in filesToDelete:
 615                     filesToDelete.remove(path)
 616             elif modifier == "D":
 617                 filesToDelete.add(path)
 618                 if path in filesToAdd:
 619                     filesToAdd.remove(path)
 620             elif modifier == "R":
 621                 src, dest = diff['src'], diff['dst']
 622                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 623                 p4_system("edit \"%s\"" % (dest))
 624                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 625                     filesToChangeExecBit[dest] = diff['dst_mode']
 626                 os.unlink(dest)
 627                 editedFiles.add(dest)
 628                 filesToDelete.add(src)
 629             else:
 630                 die("unknown modifier %s for %s" % (modifier, path))
 631
 632         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 633         patchcmd = diffcmd + " | git apply "
 634         tryPatchCmd = patchcmd + "--check -"
 635         applyPatchCmd = patchcmd + "--check --apply -"
 636
 637         if os.system(tryPatchCmd) != 0:
 638             print "Unfortunately applying the change failed!"
 639             print "What do you want to do?"
 640             response = "x"
 641             while response != "s" and response != "a" and response != "w":
 642                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 643                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 644             if response == "s":
 645                 print "Skipping! Good luck with the next patches..."
 646                 for f in editedFiles:
 647                     p4_system("revert \"%s\"" % f);
 648                 for f in filesToAdd:
 649                     system("rm %s" %f)
 650                 return
 651             elif response == "a":
 652                 os.system(applyPatchCmd)
 653                 if len(filesToAdd) > 0:
 654                     print "You may also want to call p4 add on the following files:"
 655                     print " ".join(filesToAdd)
 656                 if len(filesToDelete):
 657                     print "The following files should be scheduled for deletion with p4 delete:"
 658                     print " ".join(filesToDelete)
 659                 die("Please resolve and submit the conflict manually and "
 660                     + "continue afterwards with git-p4 submit --continue")
 661             elif response == "w":
 662                 system(diffcmd + " > patch.txt")
 663                 print "Patch saved to patch.txt in %s !" % self.clientPath
 664                 die("Please resolve and submit the conflict manually and "
 665                     "continue afterwards with git-p4 submit --continue")
 666
 667         system(applyPatchCmd)
 668
 669         for f in filesToAdd:
 670             p4_system("add \"%s\"" % f)
 671         for f in filesToDelete:
 672             p4_system("revert \"%s\"" % f)
 673             p4_system("delete \"%s\"" % f)
 674
 675         # Set/clear executable bits
 676         for f in filesToChangeExecBit.keys():
 677             mode = filesToChangeExecBit[f]
 678             setP4ExecBit(f, mode)
 679
 680         logMessage = extractLogMessageFromGitCommit(id)
 681         logMessage = logMessage.strip()
 682
 683         template = self.prepareSubmitTemplate()
 684
 685         if self.interactive:
 686             submitTemplate = self.prepareLogMessage(template, logMessage)
 687             if os.environ.has_key("P4DIFF"):
 688                 del(os.environ["P4DIFF"])
 689             diff = p4_read_pipe("diff -du ...")
 690
 691             newdiff = ""
 692             for newFile in filesToAdd:
 693                 newdiff += "==== new file ====\n"
 694                 newdiff += "--- /dev/null\n"
 695                 newdiff += "+++ %s\n" % newFile
 696                 f = open(newFile, "r")
 697                 for line in f.readlines():
 698                     newdiff += "+" + line
 699                 f.close()
 700
 701             separatorLine = "######## everything below this line is just the diff #######\n"
 702
 703             [handle, fileName] = tempfile.mkstemp()
 704             tmpFile = os.fdopen(handle, "w+")
 705             if self.isWindows:
 706                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 707                 separatorLine = separatorLine.replace("\n", "\r\n")
 708                 newdiff = newdiff.replace("\n", "\r\n")
 709             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 710             tmpFile.close()
 711             mtime = os.stat(fileName).st_mtime
 712             defaultEditor = "vi"
 713             if platform.system() == "Windows":
 714                 defaultEditor = "notepad"
 715             if os.environ.has_key("P4EDITOR"):
 716                 editor = os.environ.get("P4EDITOR")
 717             else:
 718                 editor = os.environ.get("EDITOR", defaultEditor);
 719             system(editor + " " + fileName)
 720
 721             response = "y"
 722             if os.stat(fileName).st_mtime <= mtime:
 723                 response = "x"
 724                 while response != "y" and response != "n":
 725                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 726
 727             if response == "y":
 728                 tmpFile = open(fileName, "rb")
 729                 message = tmpFile.read()
 730                 tmpFile.close()
 731                 submitTemplate = message[:message.index(separatorLine)]
 732                 if self.isWindows:
 733                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 734                 p4_write_pipe("submit -i", submitTemplate)
 735             else:
 736                 for f in editedFiles:
 737                     p4_system("revert \"%s\"" % f);
 738                 for f in filesToAdd:
 739                     p4_system("revert \"%s\"" % f);
 740                     system("rm %s" %f)
 741
 742             os.remove(fileName)
 743         else:
 744             fileName = "submit.txt"
 745             file = open(fileName, "w+")
 746             file.write(self.prepareLogMessage(template, logMessage))
 747             file.close()
 748             print ("Perforce submit template written as %s. "
 749                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 750                    % (fileName, fileName))
 751
 752     def run(self, args):
 753         if len(args) == 0:
 754             self.master = currentGitBranch()
 755             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 756                 die("Detecting current git branch failed!")
 757         elif len(args) == 1:
 758             self.master = args[0]
 759         else:
 760             return False
 761
 762         allowSubmit = gitConfig("git-p4.allowSubmit")
 763         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 764             die("%s is not in git-p4.allowSubmit" % self.master)
 765
 766         [upstream, settings] = findUpstreamBranchPoint()
 767         self.depotPath = settings['depot-paths'][0]
 768         if len(self.origin) == 0:
 769             self.origin = upstream
 770
 771         if self.verbose:
 772             print "Origin branch is " + self.origin
 773
 774         if len(self.depotPath) == 0:
 775             print "Internal error: cannot locate perforce depot path from existing branches"
 776             sys.exit(128)
 777
 778         self.clientPath = p4Where(self.depotPath)
 779
 780         if len(self.clientPath) == 0:
 781             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 782             sys.exit(128)
 783
 784         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 785         self.oldWorkingDirectory = os.getcwd()
 786
 787         chdir(self.clientPath)
 788         print "Syncronizing p4 checkout..."
 789         p4_system("sync ...")
 790
 791         self.check()
 792
 793         commits = []
 794         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 795             commits.append(line.strip())
 796         commits.reverse()
 797
 798         while len(commits) > 0:
 799             commit = commits[0]
 800             commits = commits[1:]
 801             self.applyCommit(commit)
 802             if not self.interactive:
 803                 break
 804
 805         if len(commits) == 0:
 806             print "All changes applied!"
 807             chdir(self.oldWorkingDirectory)
 808
 809             sync = P4Sync()
 810             sync.run([])
 811
 812             rebase = P4Rebase()
 813             rebase.rebase()
 814
 815         return True
 816
 817 class P4Sync(Command):
 818     def __init__(self):
 819         Command.__init__(self)
 820         self.options = [
 821                 optparse.make_option("--branch", dest="branch"),
 822                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 823                 optparse.make_option("--changesfile", dest="changesFile"),
 824                 optparse.make_option("--silent", dest="silent", action="store_true"),
 825                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 826                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 827                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 828                                      help="Import into refs/heads/ , not refs/remotes"),
 829                 optparse.make_option("--max-changes", dest="maxChanges"),
 830                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 831                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 832                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 833                                      help="Only sync files that are included in the Perforce Client Spec")
 834         ]
 835         self.description = """Imports from Perforce into a git repository.\n
 836     example:
 837     //depot/my/project/ -- to import the current head
 838     //depot/my/project/@all -- to import everything
 839     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 840
 841     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 842
 843         self.usage += " //depot/path[@revRange]"
 844         self.silent = False
 845         self.createdBranches = Set()
 846         self.committedChanges = Set()
 847         self.branch = ""
 848         self.detectBranches = False
 849         self.detectLabels = False
 850         self.changesFile = ""
 851         self.syncWithOrigin = True
 852         self.verbose = False
 853         self.importIntoRemotes = True
 854         self.maxChanges = ""
 855         self.isWindows = (platform.system() == "Windows")
 856         self.keepRepoPath = False
 857         self.depotPaths = None
 858         self.p4BranchesInGit = []
 859         self.cloneExclude = []
 860         self.useClientSpec = False
 861         self.clientSpecDirs = []
 862
 863         if gitConfig("git-p4.syncFromOrigin") == "false":
 864             self.syncWithOrigin = False
 865
 866     def extractFilesFromCommit(self, commit):
 867         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 868                              for path in self.cloneExclude]
 869         files = []
 870         fnum = 0
 871         while commit.has_key("depotFile%s" % fnum):
 872             path =  commit["depotFile%s" % fnum]
 873
 874             if [p for p in self.cloneExclude
 875                 if path.startswith (p)]:
 876                 found = False
 877             else:
 878                 found = [p for p in self.depotPaths
 879                          if path.startswith (p)]
 880             if not found:
 881                 fnum = fnum + 1
 882                 continue
 883
 884             file = {}
 885             file["path"] = path
 886             file["rev"] = commit["rev%s" % fnum]
 887             file["action"] = commit["action%s" % fnum]
 888             file["type"] = commit["type%s" % fnum]
 889             files.append(file)
 890             fnum = fnum + 1
 891         return files
 892
 893     def stripRepoPath(self, path, prefixes):
 894         if self.keepRepoPath:
 895             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 896
 897         for p in prefixes:
 898             if path.startswith(p):
 899                 path = path[len(p):]
 900
 901         return path
 902
 903     def splitFilesIntoBranches(self, commit):
 904         branches = {}
 905         fnum = 0
 906         while commit.has_key("depotFile%s" % fnum):
 907             path =  commit["depotFile%s" % fnum]
 908             found = [p for p in self.depotPaths
 909                      if path.startswith (p)]
 910             if not found:
 911                 fnum = fnum + 1
 912                 continue
 913
 914             file = {}
 915             file["path"] = path
 916             file["rev"] = commit["rev%s" % fnum]
 917             file["action"] = commit["action%s" % fnum]
 918             file["type"] = commit["type%s" % fnum]
 919             fnum = fnum + 1
 920
 921             relPath = self.stripRepoPath(path, self.depotPaths)
 922
 923             for branch in self.knownBranches.keys():
 924
 925                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 926                 if relPath.startswith(branch + "/"):
 927                     if branch not in branches:
 928                         branches[branch] = []
 929                     branches[branch].append(file)
 930                     break
 931
 932         return branches
 933
 934     ## Should move this out, doesn't use SELF.
 935     def readP4Files(self, files):
 936         filesForCommit = []
 937         filesToRead = []
 938
 939         for f in files:
 940             includeFile = True
 941             for val in self.clientSpecDirs:
 942                 if f['path'].startswith(val[0]):
 943                     if val[1] <= 0:
 944                         includeFile = False
 945                     break
 946
 947             if includeFile:
 948                 filesForCommit.append(f)
 949                 if f['action'] != 'delete':
 950                     filesToRead.append(f)
 951
 952         filedata = []
 953         if len(filesToRead) > 0:
 954             filedata = p4CmdList('-x - print',
 955                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 956                                                   for f in filesToRead]),
 957                                  stdin_mode='w+')
 958
 959             if "p4ExitCode" in filedata[0]:
 960                 die("Problems executing p4. Error: [%d]."
 961                     % (filedata[0]['p4ExitCode']));
 962
 963         j = 0;
 964         contents = {}
 965         while j < len(filedata):
 966             stat = filedata[j]
 967             j += 1
 968             text = [];
 969             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 970                 text.append(filedata[j]['data'])
 971                 j += 1
 972             text = ''.join(text)
 973
 974             if not stat.has_key('depotFile'):
 975                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 976                 continue
 977
 978             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 979                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 980             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 981                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 982
 983             contents[stat['depotFile']] = text
 984
 985         for f in filesForCommit:
 986             path = f['path']
 987             if contents.has_key(path):
 988                 f['data'] = contents[path]
 989
 990         return filesForCommit
 991
 992     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 993         epoch = details["time"]
 994         author = details["user"]
 995
 996         if self.verbose:
 997             print "commit into %s" % branch
 998
 999         # start with reading files; if that fails, we should not
1000         # create a commit.
1001         new_files = []
1002         for f in files:
1003             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1004                 new_files.append (f)
1005             else:
1006                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1007         files = self.readP4Files(new_files)
1008
1009         self.gitStream.write("commit %s\n" % branch)
1010 #        gitStream.write("mark :%s\n" % details["change"])
1011         self.committedChanges.add(int(details["change"]))
1012         committer = ""
1013         if author not in self.users:
1014             self.getUserMapFromPerforceServer()
1015         if author in self.users:
1016             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1017         else:
1018             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1019
1020         self.gitStream.write("committer %s\n" % committer)
1021
1022         self.gitStream.write("data <<EOT\n")
1023         self.gitStream.write(details["desc"])
1024         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1025                              % (','.join (branchPrefixes), details["change"]))
1026         if len(details['options']) > 0:
1027             self.gitStream.write(": options = %s" % details['options'])
1028         self.gitStream.write("]\nEOT\n\n")
1029
1030         if len(parent) > 0:
1031             if self.verbose:
1032                 print "parent %s" % parent
1033             self.gitStream.write("from %s\n" % parent)
1034
1035         for file in files:
1036             if file["type"] == "apple":
1037                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1038                 continue
1039
1040             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1041             if file["action"] == "delete":
1042                 self.gitStream.write("D %s\n" % relPath)
1043             else:
1044                 data = file['data']
1045
1046                 mode = "644"
1047                 if isP4Exec(file["type"]):
1048                     mode = "755"
1049                 elif file["type"] == "symlink":
1050                     mode = "120000"
1051                     # p4 print on a symlink contains "target\n", so strip it off
1052                     data = data[:-1]
1053
1054                 if self.isWindows and file["type"].endswith("text"):
1055                     data = data.replace("\r\n", "\n")
1056
1057                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1058                 self.gitStream.write("data %s\n" % len(data))
1059                 self.gitStream.write(data)
1060                 self.gitStream.write("\n")
1061
1062         self.gitStream.write("\n")
1063
1064         change = int(details["change"])
1065
1066         if self.labels.has_key(change):
1067             label = self.labels[change]
1068             labelDetails = label[0]
1069             labelRevisions = label[1]
1070             if self.verbose:
1071                 print "Change %s is labelled %s" % (change, labelDetails)
1072
1073             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1074                                                     for p in branchPrefixes]))
1075
1076             if len(files) == len(labelRevisions):
1077
1078                 cleanedFiles = {}
1079                 for info in files:
1080                     if info["action"] == "delete":
1081                         continue
1082                     cleanedFiles[info["depotFile"]] = info["rev"]
1083
1084                 if cleanedFiles == labelRevisions:
1085                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1086                     self.gitStream.write("from %s\n" % branch)
1087
1088                     owner = labelDetails["Owner"]
1089                     tagger = ""
1090                     if author in self.users:
1091                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1092                     else:
1093                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1094                     self.gitStream.write("tagger %s\n" % tagger)
1095                     self.gitStream.write("data <<EOT\n")
1096                     self.gitStream.write(labelDetails["Description"])
1097                     self.gitStream.write("EOT\n\n")
1098
1099                 else:
1100                     if not self.silent:
1101                         print ("Tag %s does not match with change %s: files do not match."
1102                                % (labelDetails["label"], change))
1103
1104             else:
1105                 if not self.silent:
1106                     print ("Tag %s does not match with change %s: file count is different."
1107                            % (labelDetails["label"], change))
1108
1109     def getUserCacheFilename(self):
1110         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1111         return home + "/.gitp4-usercache.txt"
1112
1113     def getUserMapFromPerforceServer(self):
1114         if self.userMapFromPerforceServer:
1115             return
1116         self.users = {}
1117
1118         for output in p4CmdList("users"):
1119             if not output.has_key("User"):
1120                 continue
1121             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1122
1123
1124         s = ''
1125         for (key, val) in self.users.items():
1126             s += "%s\t%s\n" % (key, val)
1127
1128         open(self.getUserCacheFilename(), "wb").write(s)
1129         self.userMapFromPerforceServer = True
1130
1131     def loadUserMapFromCache(self):
1132         self.users = {}
1133         self.userMapFromPerforceServer = False
1134         try:
1135             cache = open(self.getUserCacheFilename(), "rb")
1136             lines = cache.readlines()
1137             cache.close()
1138             for line in lines:
1139                 entry = line.strip().split("\t")
1140                 self.users[entry[0]] = entry[1]
1141         except IOError:
1142             self.getUserMapFromPerforceServer()
1143
1144     def getLabels(self):
1145         self.labels = {}
1146
1147         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1148         if len(l) > 0 and not self.silent:
1149             print "Finding files belonging to labels in %s" % `self.depotPaths`
1150
1151         for output in l:
1152             label = output["label"]
1153             revisions = {}
1154             newestChange = 0
1155             if self.verbose:
1156                 print "Querying files for label %s" % label
1157             for file in p4CmdList("files "
1158                                   +  ' '.join (["%s...@%s" % (p, label)
1159                                                 for p in self.depotPaths])):
1160                 revisions[file["depotFile"]] = file["rev"]
1161                 change = int(file["change"])
1162                 if change > newestChange:
1163                     newestChange = change
1164
1165             self.labels[newestChange] = [output, revisions]
1166
1167         if self.verbose:
1168             print "Label changes: %s" % self.labels.keys()
1169
1170     def guessProjectName(self):
1171         for p in self.depotPaths:
1172             if p.endswith("/"):
1173                 p = p[:-1]
1174             p = p[p.strip().rfind("/") + 1:]
1175             if not p.endswith("/"):
1176                p += "/"
1177             return p
1178
1179     def getBranchMapping(self):
1180         lostAndFoundBranches = set()
1181
1182         for info in p4CmdList("branches"):
1183             details = p4Cmd("branch -o %s" % info["branch"])
1184             viewIdx = 0
1185             while details.has_key("View%s" % viewIdx):
1186                 paths = details["View%s" % viewIdx].split(" ")
1187                 viewIdx = viewIdx + 1
1188                 # require standard //depot/foo/... //depot/bar/... mapping
1189                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1190                     continue
1191                 source = paths[0]
1192                 destination = paths[1]
1193                 ## HACK
1194                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1195                     source = source[len(self.depotPaths[0]):-4]
1196                     destination = destination[len(self.depotPaths[0]):-4]
1197
1198                     if destination in self.knownBranches:
1199                         if not self.silent:
1200                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1201                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1202                         continue
1203
1204                     self.knownBranches[destination] = source
1205
1206                     lostAndFoundBranches.discard(destination)
1207
1208                     if source not in self.knownBranches:
1209                         lostAndFoundBranches.add(source)
1210
1211
1212         for branch in lostAndFoundBranches:
1213             self.knownBranches[branch] = branch
1214
1215     def getBranchMappingFromGitBranches(self):
1216         branches = p4BranchesInGit(self.importIntoRemotes)
1217         for branch in branches.keys():
1218             if branch == "master":
1219                 branch = "main"
1220             else:
1221                 branch = branch[len(self.projectName):]
1222             self.knownBranches[branch] = branch
1223
1224     def listExistingP4GitBranches(self):
1225         # branches holds mapping from name to commit
1226         branches = p4BranchesInGit(self.importIntoRemotes)
1227         self.p4BranchesInGit = branches.keys()
1228         for branch in branches.keys():
1229             self.initialParents[self.refPrefix + branch] = branches[branch]
1230
1231     def updateOptionDict(self, d):
1232         option_keys = {}
1233         if self.keepRepoPath:
1234             option_keys['keepRepoPath'] = 1
1235
1236         d["options"] = ' '.join(sorted(option_keys.keys()))
1237
1238     def readOptions(self, d):
1239         self.keepRepoPath = (d.has_key('options')
1240                              and ('keepRepoPath' in d['options']))
1241
1242     def gitRefForBranch(self, branch):
1243         if branch == "main":
1244             return self.refPrefix + "master"
1245
1246         if len(branch) <= 0:
1247             return branch
1248
1249         return self.refPrefix + self.projectName + branch
1250
1251     def gitCommitByP4Change(self, ref, change):
1252         if self.verbose:
1253             print "looking in ref " + ref + " for change %s using bisect..." % change
1254
1255         earliestCommit = ""
1256         latestCommit = parseRevision(ref)
1257
1258         while True:
1259             if self.verbose:
1260                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1261             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1262             if len(next) == 0:
1263                 if self.verbose:
1264                     print "argh"
1265                 return ""
1266             log = extractLogMessageFromGitCommit(next)
1267             settings = extractSettingsGitLog(log)
1268             currentChange = int(settings['change'])
1269             if self.verbose:
1270                 print "current change %s" % currentChange
1271
1272             if currentChange == change:
1273                 if self.verbose:
1274                     print "found %s" % next
1275                 return next
1276
1277             if currentChange < change:
1278                 earliestCommit = "^%s" % next
1279             else:
1280                 latestCommit = "%s" % next
1281
1282         return ""
1283
1284     def importNewBranch(self, branch, maxChange):
1285         # make fast-import flush all changes to disk and update the refs using the checkpoint
1286         # command so that we can try to find the branch parent in the git history
1287         self.gitStream.write("checkpoint\n\n");
1288         self.gitStream.flush();
1289         branchPrefix = self.depotPaths[0] + branch + "/"
1290         range = "@1,%s" % maxChange
1291         #print "prefix" + branchPrefix
1292         changes = p4ChangesForPaths([branchPrefix], range)
1293         if len(changes) <= 0:
1294             return False
1295         firstChange = changes[0]
1296         #print "first change in branch: %s" % firstChange
1297         sourceBranch = self.knownBranches[branch]
1298         sourceDepotPath = self.depotPaths[0] + sourceBranch
1299         sourceRef = self.gitRefForBranch(sourceBranch)
1300         #print "source " + sourceBranch
1301
1302         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1303         #print "branch parent: %s" % branchParentChange
1304         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1305         if len(gitParent) > 0:
1306             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1307             #print "parent git commit: %s" % gitParent
1308
1309         self.importChanges(changes)
1310         return True
1311
1312     def importChanges(self, changes):
1313         cnt = 1
1314         for change in changes:
1315             description = p4Cmd("describe %s" % change)
1316             self.updateOptionDict(description)
1317
1318             if not self.silent:
1319                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1320                 sys.stdout.flush()
1321             cnt = cnt + 1
1322
1323             try:
1324                 if self.detectBranches:
1325                     branches = self.splitFilesIntoBranches(description)
1326                     for branch in branches.keys():
1327                         ## HACK  --hwn
1328                         branchPrefix = self.depotPaths[0] + branch + "/"
1329
1330                         parent = ""
1331
1332                         filesForCommit = branches[branch]
1333
1334                         if self.verbose:
1335                             print "branch is %s" % branch
1336
1337                         self.updatedBranches.add(branch)
1338
1339                         if branch not in self.createdBranches:
1340                             self.createdBranches.add(branch)
1341                             parent = self.knownBranches[branch]
1342                             if parent == branch:
1343                                 parent = ""
1344                             else:
1345                                 fullBranch = self.projectName + branch
1346                                 if fullBranch not in self.p4BranchesInGit:
1347                                     if not self.silent:
1348                                         print("\n    Importing new branch %s" % fullBranch);
1349                                     if self.importNewBranch(branch, change - 1):
1350                                         parent = ""
1351                                         self.p4BranchesInGit.append(fullBranch)
1352                                     if not self.silent:
1353                                         print("\n    Resuming with change %s" % change);
1354
1355                                 if self.verbose:
1356                                     print "parent determined through known branches: %s" % parent
1357
1358                         branch = self.gitRefForBranch(branch)
1359                         parent = self.gitRefForBranch(parent)
1360
1361                         if self.verbose:
1362                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1363
1364                         if len(parent) == 0 and branch in self.initialParents:
1365                             parent = self.initialParents[branch]
1366                             del self.initialParents[branch]
1367
1368                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1369                 else:
1370                     files = self.extractFilesFromCommit(description)
1371                     self.commit(description, files, self.branch, self.depotPaths,
1372                                 self.initialParent)
1373                     self.initialParent = ""
1374             except IOError:
1375                 print self.gitError.read()
1376                 sys.exit(1)
1377
1378     def importHeadRevision(self, revision):
1379         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1380
1381         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1382         details["desc"] = ("Initial import of %s from the state at revision %s"
1383                            % (' '.join(self.depotPaths), revision))
1384         details["change"] = revision
1385         newestRevision = 0
1386
1387         fileCnt = 0
1388         for info in p4CmdList("files "
1389                               +  ' '.join(["%s...%s"
1390                                            % (p, revision)
1391                                            for p in self.depotPaths])):
1392
1393             if info['code'] == 'error':
1394                 sys.stderr.write("p4 returned an error: %s\n"
1395                                  % info['data'])
1396                 sys.exit(1)
1397
1398
1399             change = int(info["change"])
1400             if change > newestRevision:
1401                 newestRevision = change
1402
1403             if info["action"] == "delete":
1404                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1405                 #fileCnt = fileCnt + 1
1406                 continue
1407
1408             for prop in ["depotFile", "rev", "action", "type" ]:
1409                 details["%s%s" % (prop, fileCnt)] = info[prop]
1410
1411             fileCnt = fileCnt + 1
1412
1413         details["change"] = newestRevision
1414         self.updateOptionDict(details)
1415         try:
1416             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1417         except IOError:
1418             print "IO error with git fast-import. Is your git version recent enough?"
1419             print self.gitError.read()
1420
1421
1422     def getClientSpec(self):
1423         specList = p4CmdList( "client -o" )
1424         temp = {}
1425         for entry in specList:
1426             for k,v in entry.iteritems():
1427                 if k.startswith("View"):
1428                     if v.startswith('"'):
1429                         start = 1
1430                     else:
1431                         start = 0
1432                     index = v.find("...")
1433                     v = v[start:index]
1434                     if v.startswith("-"):
1435                         v = v[1:]
1436                         temp[v] = -len(v)
1437                     else:
1438                         temp[v] = len(v)
1439         self.clientSpecDirs = temp.items()
1440         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1441
1442     def run(self, args):
1443         self.depotPaths = []
1444         self.changeRange = ""
1445         self.initialParent = ""
1446         self.previousDepotPaths = []
1447
1448         # map from branch depot path to parent branch
1449         self.knownBranches = {}
1450         self.initialParents = {}
1451         self.hasOrigin = originP4BranchesExist()
1452         if not self.syncWithOrigin:
1453             self.hasOrigin = False
1454
1455         if self.importIntoRemotes:
1456             self.refPrefix = "refs/remotes/p4/"
1457         else:
1458             self.refPrefix = "refs/heads/p4/"
1459
1460         if self.syncWithOrigin and self.hasOrigin:
1461             if not self.silent:
1462                 print "Syncing with origin first by calling git fetch origin"
1463             system("git fetch origin")
1464
1465         if len(self.branch) == 0:
1466             self.branch = self.refPrefix + "master"
1467             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1468                 system("git update-ref %s refs/heads/p4" % self.branch)
1469                 system("git branch -D p4");
1470             # create it /after/ importing, when master exists
1471             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1472                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1473
1474         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1475             self.getClientSpec()
1476
1477         # TODO: should always look at previous commits,
1478         # merge with previous imports, if possible.
1479         if args == []:
1480             if self.hasOrigin:
1481                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1482             self.listExistingP4GitBranches()
1483
1484             if len(self.p4BranchesInGit) > 1:
1485                 if not self.silent:
1486                     print "Importing from/into multiple branches"
1487                 self.detectBranches = True
1488
1489             if self.verbose:
1490                 print "branches: %s" % self.p4BranchesInGit
1491
1492             p4Change = 0
1493             for branch in self.p4BranchesInGit:
1494                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1495
1496                 settings = extractSettingsGitLog(logMsg)
1497
1498                 self.readOptions(settings)
1499                 if (settings.has_key('depot-paths')
1500                     and settings.has_key ('change')):
1501                     change = int(settings['change']) + 1
1502                     p4Change = max(p4Change, change)
1503
1504                     depotPaths = sorted(settings['depot-paths'])
1505                     if self.previousDepotPaths == []:
1506                         self.previousDepotPaths = depotPaths
1507                     else:
1508                         paths = []
1509                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1510                             for i in range(0, min(len(cur), len(prev))):
1511                                 if cur[i] <> prev[i]:
1512                                     i = i - 1
1513                                     break
1514
1515                             paths.append (cur[:i + 1])
1516
1517                         self.previousDepotPaths = paths
1518
1519             if p4Change > 0:
1520                 self.depotPaths = sorted(self.previousDepotPaths)
1521                 self.changeRange = "@%s,#head" % p4Change
1522                 if not self.detectBranches:
1523                     self.initialParent = parseRevision(self.branch)
1524                 if not self.silent and not self.detectBranches:
1525                     print "Performing incremental import into %s git branch" % self.branch
1526
1527         if not self.branch.startswith("refs/"):
1528             self.branch = "refs/heads/" + self.branch
1529
1530         if len(args) == 0 and self.depotPaths:
1531             if not self.silent:
1532                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1533         else:
1534             if self.depotPaths and self.depotPaths != args:
1535                 print ("previous import used depot path %s and now %s was specified. "
1536                        "This doesn't work!" % (' '.join (self.depotPaths),
1537                                                ' '.join (args)))
1538                 sys.exit(1)
1539
1540             self.depotPaths = sorted(args)
1541
1542         revision = ""
1543         self.users = {}
1544
1545         newPaths = []
1546         for p in self.depotPaths:
1547             if p.find("@") != -1:
1548                 atIdx = p.index("@")
1549                 self.changeRange = p[atIdx:]
1550                 if self.changeRange == "@all":
1551                     self.changeRange = ""
1552                 elif ',' not in self.changeRange:
1553                     revision = self.changeRange
1554                     self.changeRange = ""
1555                 p = p[:atIdx]
1556             elif p.find("#") != -1:
1557                 hashIdx = p.index("#")
1558                 revision = p[hashIdx:]
1559                 p = p[:hashIdx]
1560             elif self.previousDepotPaths == []:
1561                 revision = "#head"
1562
1563             p = re.sub ("\.\.\.$", "", p)
1564             if not p.endswith("/"):
1565                 p += "/"
1566
1567             newPaths.append(p)
1568
1569         self.depotPaths = newPaths
1570
1571
1572         self.loadUserMapFromCache()
1573         self.labels = {}
1574         if self.detectLabels:
1575             self.getLabels();
1576
1577         if self.detectBranches:
1578             ## FIXME - what's a P4 projectName ?
1579             self.projectName = self.guessProjectName()
1580
1581             if self.hasOrigin:
1582                 self.getBranchMappingFromGitBranches()
1583             else:
1584                 self.getBranchMapping()
1585             if self.verbose:
1586                 print "p4-git branches: %s" % self.p4BranchesInGit
1587                 print "initial parents: %s" % self.initialParents
1588             for b in self.p4BranchesInGit:
1589                 if b != "master":
1590
1591                     ## FIXME
1592                     b = b[len(self.projectName):]
1593                 self.createdBranches.add(b)
1594
1595         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1596
1597         importProcess = subprocess.Popen(["git", "fast-import"],
1598                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1599                                          stderr=subprocess.PIPE);
1600         self.gitOutput = importProcess.stdout
1601         self.gitStream = importProcess.stdin
1602         self.gitError = importProcess.stderr
1603
1604         if revision:
1605             self.importHeadRevision(revision)
1606         else:
1607             changes = []
1608
1609             if len(self.changesFile) > 0:
1610                 output = open(self.changesFile).readlines()
1611                 changeSet = Set()
1612                 for line in output:
1613                     changeSet.add(int(line))
1614
1615                 for change in changeSet:
1616                     changes.append(change)
1617
1618                 changes.sort()
1619             else:
1620                 if self.verbose:
1621                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1622                                                               self.changeRange)
1623                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1624
1625                 if len(self.maxChanges) > 0:
1626                     changes = changes[:min(int(self.maxChanges), len(changes))]
1627
1628             if len(changes) == 0:
1629                 if not self.silent:
1630                     print "No changes to import!"
1631                 return True
1632
1633             if not self.silent and not self.detectBranches:
1634                 print "Import destination: %s" % self.branch
1635
1636             self.updatedBranches = set()
1637
1638             self.importChanges(changes)
1639
1640             if not self.silent:
1641                 print ""
1642                 if len(self.updatedBranches) > 0:
1643                     sys.stdout.write("Updated branches: ")
1644                     for b in self.updatedBranches:
1645                         sys.stdout.write("%s " % b)
1646                     sys.stdout.write("\n")
1647
1648         self.gitStream.close()
1649         if importProcess.wait() != 0:
1650             die("fast-import failed: %s" % self.gitError.read())
1651         self.gitOutput.close()
1652         self.gitError.close()
1653
1654         return True
1655
1656 class P4Rebase(Command):
1657     def __init__(self):
1658         Command.__init__(self)
1659         self.options = [ ]
1660         self.description = ("Fetches the latest revision from perforce and "
1661                             + "rebases the current work (branch) against it")
1662         self.verbose = False
1663
1664     def run(self, args):
1665         sync = P4Sync()
1666         sync.run([])
1667
1668         return self.rebase()
1669
1670     def rebase(self):
1671         if os.system("git update-index --refresh") != 0:
1672             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1673         if len(read_pipe("git diff-index HEAD --")) > 0:
1674             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1675
1676         [upstream, settings] = findUpstreamBranchPoint()
1677         if len(upstream) == 0:
1678             die("Cannot find upstream branchpoint for rebase")
1679
1680         # the branchpoint may be p4/foo~3, so strip off the parent
1681         upstream = re.sub("~[0-9]+$", "", upstream)
1682
1683         print "Rebasing the current branch onto %s" % upstream
1684         oldHead = read_pipe("git rev-parse HEAD").strip()
1685         system("git rebase %s" % upstream)
1686         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1687         return True
1688
1689 class P4Clone(P4Sync):
1690     def __init__(self):
1691         P4Sync.__init__(self)
1692         self.description = "Creates a new git repository and imports from Perforce into it"
1693         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1694         self.options += [
1695             optparse.make_option("--destination", dest="cloneDestination",
1696                                  action='store', default=None,
1697                                  help="where to leave result of the clone"),
1698             optparse.make_option("-/", dest="cloneExclude",
1699                                  action="append", type="string",
1700                                  help="exclude depot path")
1701         ]
1702         self.cloneDestination = None
1703         self.needsGit = False
1704
1705     # This is required for the "append" cloneExclude action
1706     def ensure_value(self, attr, value):
1707         if not hasattr(self, attr) or getattr(self, attr) is None:
1708             setattr(self, attr, value)
1709         return getattr(self, attr)
1710
1711     def defaultDestination(self, args):
1712         ## TODO: use common prefix of args?
1713         depotPath = args[0]
1714         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1715         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1716         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1717         depotDir = re.sub(r"/$", "", depotDir)
1718         return os.path.split(depotDir)[1]
1719
1720     def run(self, args):
1721         if len(args) < 1:
1722             return False
1723
1724         if self.keepRepoPath and not self.cloneDestination:
1725             sys.stderr.write("Must specify destination for --keep-path\n")
1726             sys.exit(1)
1727
1728         depotPaths = args
1729
1730         if not self.cloneDestination and len(depotPaths) > 1:
1731             self.cloneDestination = depotPaths[-1]
1732             depotPaths = depotPaths[:-1]
1733
1734         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1735         for p in depotPaths:
1736             if not p.startswith("//"):
1737                 return False
1738
1739         if not self.cloneDestination:
1740             self.cloneDestination = self.defaultDestination(args)
1741
1742         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1743         if not os.path.exists(self.cloneDestination):
1744             os.makedirs(self.cloneDestination)
1745         chdir(self.cloneDestination)
1746         system("git init")
1747         self.gitdir = os.getcwd() + "/.git"
1748         if not P4Sync.run(self, depotPaths):
1749             return False
1750         if self.branch != "master":
1751             if self.importIntoRemotes:
1752                 masterbranch = "refs/remotes/p4/master"
1753             else:
1754                 masterbranch = "refs/heads/p4/master"
1755             if gitBranchExists(masterbranch):
1756                 system("git branch master %s" % masterbranch)
1757                 system("git checkout -f")
1758             else:
1759                 print "Could not detect main branch. No checkout/master branch created."
1760
1761         return True
1762
1763 class P4Branches(Command):
1764     def __init__(self):
1765         Command.__init__(self)
1766         self.options = [ ]
1767         self.description = ("Shows the git branches that hold imports and their "
1768                             + "corresponding perforce depot paths")
1769         self.verbose = False
1770
1771     def run(self, args):
1772         if originP4BranchesExist():
1773             createOrUpdateBranchesFromOrigin()
1774
1775         cmdline = "git rev-parse --symbolic "
1776         cmdline += " --remotes"
1777
1778         for line in read_pipe_lines(cmdline):
1779             line = line.strip()
1780
1781             if not line.startswith('p4/') or line == "p4/HEAD":
1782                 continue
1783             branch = line
1784
1785             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1786             settings = extractSettingsGitLog(log)
1787
1788             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1789         return True
1790
1791 class HelpFormatter(optparse.IndentedHelpFormatter):
1792     def __init__(self):
1793         optparse.IndentedHelpFormatter.__init__(self)
1794
1795     def format_description(self, description):
1796         if description:
1797             return description + "\n"
1798         else:
1799             return ""
1800
1801 def printUsage(commands):
1802     print "usage: %s <command> [options]" % sys.argv[0]
1803     print ""
1804     print "valid commands: %s" % ", ".join(commands)
1805     print ""
1806     print "Try %s <command> --help for command specific help." % sys.argv[0]
1807     print ""
1808
1809 commands = {
1810     "debug" : P4Debug,
1811     "submit" : P4Submit,
1812     "commit" : P4Submit,
1813     "sync" : P4Sync,
1814     "rebase" : P4Rebase,
1815     "clone" : P4Clone,
1816     "rollback" : P4RollBack,
1817     "branches" : P4Branches
1818 }
1819
1820
1821 def main():
1822     if len(sys.argv[1:]) == 0:
1823         printUsage(commands.keys())
1824         sys.exit(2)
1825
1826     cmd = ""
1827     cmdName = sys.argv[1]
1828     try:
1829         klass = commands[cmdName]
1830         cmd = klass()
1831     except KeyError:
1832         print "unknown command %s" % cmdName
1833         print ""
1834         printUsage(commands.keys())
1835         sys.exit(2)
1836
1837     options = cmd.options
1838     cmd.gitdir = os.environ.get("GIT_DIR", None)
1839
1840     args = sys.argv[2:]
1841
1842     if len(options) > 0:
1843         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1844
1845         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1846                                        options,
1847                                        description = cmd.description,
1848                                        formatter = HelpFormatter())
1849
1850         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1851     global verbose
1852     verbose = cmd.verbose
1853     if cmd.needsGit:
1854         if cmd.gitdir == None:
1855             cmd.gitdir = os.path.abspath(".git")
1856             if not isValidGitDir(cmd.gitdir):
1857                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1858                 if os.path.exists(cmd.gitdir):
1859                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1860                     if len(cdup) > 0:
1861                         chdir(cdup);
1862
1863         if not isValidGitDir(cmd.gitdir):
1864             if isValidGitDir(cmd.gitdir + "/.git"):
1865                 cmd.gitdir += "/.git"
1866             else:
1867                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1868
1869         os.environ["GIT_DIR"] = cmd.gitdir
1870
1871     if not cmd.run(args):
1872         parser.print_help()
1873
1874
1875 if __name__ == '__main__':
1876     main()