contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s" % file)
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                 optparse.make_option("--origin", dest="origin"),
 470                 optparse.make_option("--reset", action="store_true", dest="reset"),
 471                 optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 472                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 473         ]
 474         self.description = "Submit changes from git to the perforce depot."
 475         self.usage += " [name of git branch to submit into perforce depot]"
 476         self.firstTime = True
 477         self.reset = False
 478         self.interactive = True
 479         self.firstTime = True
 480         self.origin = ""
 481         self.directSubmit = False
 482         self.detectRename = False
 483         self.verbose = False
 484         self.isWindows = (platform.system() == "Windows")
 485
 486     def check(self):
 487         if len(p4CmdList("opened ...")) > 0:
 488             die("You have files opened with perforce! Close them before starting the sync.")
 489
 490     def start(self):
 491         if len(self.config) > 0 and not self.reset:
 492             die("Cannot start sync. Previous sync config found at %s\n"
 493                 "If you want to start submitting again from scratch "
 494                 "maybe you want to call git-p4 submit --reset" % self.configFile)
 495
 496         commits = []
 497         if self.directSubmit:
 498             commits.append("0")
 499         else:
 500             for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 501                 commits.append(line.strip())
 502             commits.reverse()
 503
 504         self.config["commits"] = commits
 505
 506     # replaces everything between 'Description:' and the next P4 submit template field with the
 507     # commit message
 508     def prepareLogMessage(self, template, message):
 509         result = ""
 510
 511         inDescriptionSection = False
 512
 513         for line in template.split("\n"):
 514             if line.startswith("#"):
 515                 result += line + "\n"
 516                 continue
 517
 518             if inDescriptionSection:
 519                 if line.startswith("Files:"):
 520                     inDescriptionSection = False
 521                 else:
 522                     continue
 523             else:
 524                 if line.startswith("Description:"):
 525                     inDescriptionSection = True
 526                     line += "\n"
 527                     for messageLine in message.split("\n"):
 528                         line += "\t" + messageLine + "\n"
 529
 530             result += line + "\n"
 531
 532         return result
 533
 534     def prepareSubmitTemplate(self):
 535         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 536         template = ""
 537         inFilesSection = False
 538         for line in read_pipe_lines("p4 change -o"):
 539             if inFilesSection:
 540                 if line.startswith("\t"):
 541                     # path starts and ends with a tab
 542                     path = line[1:]
 543                     lastTab = path.rfind("\t")
 544                     if lastTab != -1:
 545                         path = path[:lastTab]
 546                         if not path.startswith(self.depotPath):
 547                             continue
 548                 else:
 549                     inFilesSection = False
 550             else:
 551                 if line.startswith("Files:"):
 552                     inFilesSection = True
 553
 554             template += line
 555
 556         return template
 557
 558     def applyCommit(self, id):
 559         if self.directSubmit:
 560             print "Applying local change in working directory/index"
 561             diff = self.diffStatus
 562         else:
 563             print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 564             diffOpts = ("", "-M")[self.detectRename]
 565             diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 566         filesToAdd = set()
 567         filesToDelete = set()
 568         editedFiles = set()
 569         filesToChangeExecBit = {}
 570         for line in diff:
 571             diff = parseDiffTreeEntry(line)
 572             modifier = diff['status']
 573             path = diff['src']
 574             if modifier == "M":
 575                 system("p4 edit \"%s\"" % path)
 576                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 577                     filesToChangeExecBit[path] = diff['dst_mode']
 578                 editedFiles.add(path)
 579             elif modifier == "A":
 580                 filesToAdd.add(path)
 581                 filesToChangeExecBit[path] = diff['dst_mode']
 582                 if path in filesToDelete:
 583                     filesToDelete.remove(path)
 584             elif modifier == "D":
 585                 filesToDelete.add(path)
 586                 if path in filesToAdd:
 587                     filesToAdd.remove(path)
 588             elif modifier == "R":
 589                 src, dest = diff['src'], diff['dst']
 590                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 591                 system("p4 edit \"%s\"" % (dest))
 592                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 593                     filesToChangeExecBit[dest] = diff['dst_mode']
 594                 os.unlink(dest)
 595                 editedFiles.add(dest)
 596                 filesToDelete.add(src)
 597             else:
 598                 die("unknown modifier %s for %s" % (modifier, path))
 599
 600         if self.directSubmit:
 601             diffcmd = "cat \"%s\"" % self.diffFile
 602         else:
 603             diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 604         patchcmd = diffcmd + " | git apply "
 605         tryPatchCmd = patchcmd + "--check -"
 606         applyPatchCmd = patchcmd + "--check --apply -"
 607
 608         if os.system(tryPatchCmd) != 0:
 609             print "Unfortunately applying the change failed!"
 610             print "What do you want to do?"
 611             response = "x"
 612             while response != "s" and response != "a" and response != "w":
 613                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 614                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 615             if response == "s":
 616                 print "Skipping! Good luck with the next patches..."
 617                 for f in editedFiles:
 618                     system("p4 revert \"%s\"" % f);
 619                 for f in filesToAdd:
 620                     system("rm %s" %f)
 621                 return
 622             elif response == "a":
 623                 os.system(applyPatchCmd)
 624                 if len(filesToAdd) > 0:
 625                     print "You may also want to call p4 add on the following files:"
 626                     print " ".join(filesToAdd)
 627                 if len(filesToDelete):
 628                     print "The following files should be scheduled for deletion with p4 delete:"
 629                     print " ".join(filesToDelete)
 630                 die("Please resolve and submit the conflict manually and "
 631                     + "continue afterwards with git-p4 submit --continue")
 632             elif response == "w":
 633                 system(diffcmd + " > patch.txt")
 634                 print "Patch saved to patch.txt in %s !" % self.clientPath
 635                 die("Please resolve and submit the conflict manually and "
 636                     "continue afterwards with git-p4 submit --continue")
 637
 638         system(applyPatchCmd)
 639
 640         for f in filesToAdd:
 641             system("p4 add \"%s\"" % f)
 642         for f in filesToDelete:
 643             system("p4 revert \"%s\"" % f)
 644             system("p4 delete \"%s\"" % f)
 645
 646         # Set/clear executable bits
 647         for f in filesToChangeExecBit.keys():
 648             mode = filesToChangeExecBit[f]
 649             setP4ExecBit(f, mode)
 650
 651         logMessage = ""
 652         if not self.directSubmit:
 653             logMessage = extractLogMessageFromGitCommit(id)
 654             if self.isWindows:
 655                 logMessage = logMessage.replace("\n", "\r\n")
 656             logMessage = logMessage.strip()
 657
 658         template = self.prepareSubmitTemplate()
 659
 660         if self.interactive:
 661             submitTemplate = self.prepareLogMessage(template, logMessage)
 662             diff = read_pipe("p4 diff -du ...")
 663
 664             for newFile in filesToAdd:
 665                 diff += "==== new file ====\n"
 666                 diff += "--- /dev/null\n"
 667                 diff += "+++ %s\n" % newFile
 668                 f = open(newFile, "r")
 669                 for line in f.readlines():
 670                     diff += "+" + line
 671                 f.close()
 672
 673             separatorLine = "######## everything below this line is just the diff #######"
 674             if platform.system() == "Windows":
 675                 separatorLine += "\r"
 676             separatorLine += "\n"
 677
 678             [handle, fileName] = tempfile.mkstemp()
 679             tmpFile = os.fdopen(handle, "w+")
 680             tmpFile.write(submitTemplate + separatorLine + diff)
 681             tmpFile.close()
 682             defaultEditor = "vi"
 683             if platform.system() == "Windows":
 684                 defaultEditor = "notepad"
 685             editor = os.environ.get("EDITOR", defaultEditor);
 686             system(editor + " " + fileName)
 687             tmpFile = open(fileName, "rb")
 688             message = tmpFile.read()
 689             tmpFile.close()
 690             os.remove(fileName)
 691             submitTemplate = message[:message.index(separatorLine)]
 692             if self.isWindows:
 693                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 694
 695             if self.directSubmit:
 696                 print "Submitting to git first"
 697                 os.chdir(self.oldWorkingDirectory)
 698                 write_pipe("git commit -a -F -", submitTemplate)
 699                 os.chdir(self.clientPath)
 700
 701             write_pipe("p4 submit -i", submitTemplate)
 702         else:
 703             fileName = "submit.txt"
 704             file = open(fileName, "w+")
 705             file.write(self.prepareLogMessage(template, logMessage))
 706             file.close()
 707             print ("Perforce submit template written as %s. "
 708                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 709                    % (fileName, fileName))
 710
 711     def run(self, args):
 712         if len(args) == 0:
 713             self.master = currentGitBranch()
 714             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 715                 die("Detecting current git branch failed!")
 716         elif len(args) == 1:
 717             self.master = args[0]
 718         else:
 719             return False
 720
 721         [upstream, settings] = findUpstreamBranchPoint()
 722         self.depotPath = settings['depot-paths'][0]
 723         if len(self.origin) == 0:
 724             self.origin = upstream
 725
 726         if self.verbose:
 727             print "Origin branch is " + self.origin
 728
 729         if len(self.depotPath) == 0:
 730             print "Internal error: cannot locate perforce depot path from existing branches"
 731             sys.exit(128)
 732
 733         self.clientPath = p4Where(self.depotPath)
 734
 735         if len(self.clientPath) == 0:
 736             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 737             sys.exit(128)
 738
 739         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 740         self.oldWorkingDirectory = os.getcwd()
 741
 742         if self.directSubmit:
 743             self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 744             if len(self.diffStatus) == 0:
 745                 print "No changes in working directory to submit."
 746                 return True
 747             patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 748             self.diffFile = self.gitdir + "/p4-git-diff"
 749             f = open(self.diffFile, "wb")
 750             f.write(patch)
 751             f.close();
 752
 753         os.chdir(self.clientPath)
 754         print "Syncronizing p4 checkout..."
 755         system("p4 sync ...")
 756
 757         if self.reset:
 758             self.firstTime = True
 759
 760         self.check()
 761         self.configFile = self.gitdir + "/p4-git-sync.cfg"
 762         self.config = shelve.open(self.configFile, writeback=True)
 763
 764         if self.firstTime:
 765             self.start()
 766
 767         commits = self.config.get("commits", [])
 768
 769         while len(commits) > 0:
 770             self.firstTime = False
 771             commit = commits[0]
 772             commits = commits[1:]
 773             self.config["commits"] = commits
 774             self.applyCommit(commit)
 775             if not self.interactive:
 776                 break
 777
 778         self.config.close()
 779
 780         if self.directSubmit:
 781             os.remove(self.diffFile)
 782
 783         if len(commits) == 0:
 784             if self.firstTime:
 785                 print "No changes found to apply between %s and current HEAD" % self.origin
 786             else:
 787                 print "All changes applied!"
 788                 os.chdir(self.oldWorkingDirectory)
 789
 790                 sync = P4Sync()
 791                 sync.run([])
 792
 793                 rebase = P4Rebase()
 794                 rebase.rebase()
 795             os.remove(self.configFile)
 796
 797         return True
 798
 799 class P4Sync(Command):
 800     def __init__(self):
 801         Command.__init__(self)
 802         self.options = [
 803                 optparse.make_option("--branch", dest="branch"),
 804                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 805                 optparse.make_option("--changesfile", dest="changesFile"),
 806                 optparse.make_option("--silent", dest="silent", action="store_true"),
 807                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 808                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 809                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 810                                      help="Import into refs/heads/ , not refs/remotes"),
 811                 optparse.make_option("--max-changes", dest="maxChanges"),
 812                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 813                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 814         ]
 815         self.description = """Imports from Perforce into a git repository.\n
 816     example:
 817     //depot/my/project/ -- to import the current head
 818     //depot/my/project/@all -- to import everything
 819     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 820
 821     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 822
 823         self.usage += " //depot/path[@revRange]"
 824         self.silent = False
 825         self.createdBranches = Set()
 826         self.committedChanges = Set()
 827         self.branch = ""
 828         self.detectBranches = False
 829         self.detectLabels = False
 830         self.changesFile = ""
 831         self.syncWithOrigin = True
 832         self.verbose = False
 833         self.importIntoRemotes = True
 834         self.maxChanges = ""
 835         self.isWindows = (platform.system() == "Windows")
 836         self.keepRepoPath = False
 837         self.depotPaths = None
 838         self.p4BranchesInGit = []
 839         self.cloneExclude = []
 840
 841         if gitConfig("git-p4.syncFromOrigin") == "false":
 842             self.syncWithOrigin = False
 843
 844     def extractFilesFromCommit(self, commit):
 845         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 846                              for path in self.cloneExclude]
 847         files = []
 848         fnum = 0
 849         while commit.has_key("depotFile%s" % fnum):
 850             path =  commit["depotFile%s" % fnum]
 851
 852             if [p for p in self.cloneExclude
 853                 if path.startswith (p)]:
 854                 found = False
 855             else:
 856                 found = [p for p in self.depotPaths
 857                          if path.startswith (p)]
 858             if not found:
 859                 fnum = fnum + 1
 860                 continue
 861
 862             file = {}
 863             file["path"] = path
 864             file["rev"] = commit["rev%s" % fnum]
 865             file["action"] = commit["action%s" % fnum]
 866             file["type"] = commit["type%s" % fnum]
 867             files.append(file)
 868             fnum = fnum + 1
 869         return files
 870
 871     def stripRepoPath(self, path, prefixes):
 872         if self.keepRepoPath:
 873             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 874
 875         for p in prefixes:
 876             if path.startswith(p):
 877                 path = path[len(p):]
 878
 879         return path
 880
 881     def splitFilesIntoBranches(self, commit):
 882         branches = {}
 883         fnum = 0
 884         while commit.has_key("depotFile%s" % fnum):
 885             path =  commit["depotFile%s" % fnum]
 886             found = [p for p in self.depotPaths
 887                      if path.startswith (p)]
 888             if not found:
 889                 fnum = fnum + 1
 890                 continue
 891
 892             file = {}
 893             file["path"] = path
 894             file["rev"] = commit["rev%s" % fnum]
 895             file["action"] = commit["action%s" % fnum]
 896             file["type"] = commit["type%s" % fnum]
 897             fnum = fnum + 1
 898
 899             relPath = self.stripRepoPath(path, self.depotPaths)
 900
 901             for branch in self.knownBranches.keys():
 902
 903                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 904                 if relPath.startswith(branch + "/"):
 905                     if branch not in branches:
 906                         branches[branch] = []
 907                     branches[branch].append(file)
 908                     break
 909
 910         return branches
 911
 912     ## Should move this out, doesn't use SELF.
 913     def readP4Files(self, files):
 914         files = [f for f in files
 915                  if f['action'] != 'delete']
 916
 917         if not files:
 918             return
 919
 920         filedata = p4CmdList('-x - print',
 921                              stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 922                                               for f in files]),
 923                              stdin_mode='w+')
 924         if "p4ExitCode" in filedata[0]:
 925             die("Problems executing p4. Error: [%d]."
 926                 % (filedata[0]['p4ExitCode']));
 927
 928         j = 0;
 929         contents = {}
 930         while j < len(filedata):
 931             stat = filedata[j]
 932             j += 1
 933             text = ''
 934             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 935                 tmp = filedata[j]['data']
 936                 if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 937                     tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 938                 elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 939                     tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 940                 text += tmp
 941                 j += 1
 942
 943
 944             if not stat.has_key('depotFile'):
 945                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 946                 continue
 947
 948             contents[stat['depotFile']] = text
 949
 950         for f in files:
 951             assert not f.has_key('data')
 952             f['data'] = contents[f['path']]
 953
 954     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 955         epoch = details["time"]
 956         author = details["user"]
 957
 958         if self.verbose:
 959             print "commit into %s" % branch
 960
 961         # start with reading files; if that fails, we should not
 962         # create a commit.
 963         new_files = []
 964         for f in files:
 965             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 966                 new_files.append (f)
 967             else:
 968                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 969         files = new_files
 970         self.readP4Files(files)
 971
 972
 973
 974
 975         self.gitStream.write("commit %s\n" % branch)
 976 #        gitStream.write("mark :%s\n" % details["change"])
 977         self.committedChanges.add(int(details["change"]))
 978         committer = ""
 979         if author not in self.users:
 980             self.getUserMapFromPerforceServer()
 981         if author in self.users:
 982             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 983         else:
 984             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 985
 986         self.gitStream.write("committer %s\n" % committer)
 987
 988         self.gitStream.write("data <<EOT\n")
 989         self.gitStream.write(details["desc"])
 990         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 991                              % (','.join (branchPrefixes), details["change"]))
 992         if len(details['options']) > 0:
 993             self.gitStream.write(": options = %s" % details['options'])
 994         self.gitStream.write("]\nEOT\n\n")
 995
 996         if len(parent) > 0:
 997             if self.verbose:
 998                 print "parent %s" % parent
 999             self.gitStream.write("from %s\n" % parent)
1000
1001         for file in files:
1002             if file["type"] == "apple":
1003                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1004                 continue
1005
1006             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1007             if file["action"] == "delete":
1008                 self.gitStream.write("D %s\n" % relPath)
1009             else:
1010                 data = file['data']
1011
1012                 mode = "644"
1013                 if isP4Exec(file["type"]):
1014                     mode = "755"
1015                 elif file["type"] == "symlink":
1016                     mode = "120000"
1017                     # p4 print on a symlink contains "target\n", so strip it off
1018                     data = data[:-1]
1019
1020                 if self.isWindows and file["type"].endswith("text"):
1021                     data = data.replace("\r\n", "\n")
1022
1023                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1024                 self.gitStream.write("data %s\n" % len(data))
1025                 self.gitStream.write(data)
1026                 self.gitStream.write("\n")
1027
1028         self.gitStream.write("\n")
1029
1030         change = int(details["change"])
1031
1032         if self.labels.has_key(change):
1033             label = self.labels[change]
1034             labelDetails = label[0]
1035             labelRevisions = label[1]
1036             if self.verbose:
1037                 print "Change %s is labelled %s" % (change, labelDetails)
1038
1039             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1040                                                     for p in branchPrefixes]))
1041
1042             if len(files) == len(labelRevisions):
1043
1044                 cleanedFiles = {}
1045                 for info in files:
1046                     if info["action"] == "delete":
1047                         continue
1048                     cleanedFiles[info["depotFile"]] = info["rev"]
1049
1050                 if cleanedFiles == labelRevisions:
1051                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1052                     self.gitStream.write("from %s\n" % branch)
1053
1054                     owner = labelDetails["Owner"]
1055                     tagger = ""
1056                     if author in self.users:
1057                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1058                     else:
1059                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1060                     self.gitStream.write("tagger %s\n" % tagger)
1061                     self.gitStream.write("data <<EOT\n")
1062                     self.gitStream.write(labelDetails["Description"])
1063                     self.gitStream.write("EOT\n\n")
1064
1065                 else:
1066                     if not self.silent:
1067                         print ("Tag %s does not match with change %s: files do not match."
1068                                % (labelDetails["label"], change))
1069
1070             else:
1071                 if not self.silent:
1072                     print ("Tag %s does not match with change %s: file count is different."
1073                            % (labelDetails["label"], change))
1074
1075     def getUserCacheFilename(self):
1076         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1077         return home + "/.gitp4-usercache.txt"
1078
1079     def getUserMapFromPerforceServer(self):
1080         if self.userMapFromPerforceServer:
1081             return
1082         self.users = {}
1083
1084         for output in p4CmdList("users"):
1085             if not output.has_key("User"):
1086                 continue
1087             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1088
1089
1090         s = ''
1091         for (key, val) in self.users.items():
1092             s += "%s\t%s\n" % (key, val)
1093
1094         open(self.getUserCacheFilename(), "wb").write(s)
1095         self.userMapFromPerforceServer = True
1096
1097     def loadUserMapFromCache(self):
1098         self.users = {}
1099         self.userMapFromPerforceServer = False
1100         try:
1101             cache = open(self.getUserCacheFilename(), "rb")
1102             lines = cache.readlines()
1103             cache.close()
1104             for line in lines:
1105                 entry = line.strip().split("\t")
1106                 self.users[entry[0]] = entry[1]
1107         except IOError:
1108             self.getUserMapFromPerforceServer()
1109
1110     def getLabels(self):
1111         self.labels = {}
1112
1113         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1114         if len(l) > 0 and not self.silent:
1115             print "Finding files belonging to labels in %s" % `self.depotPaths`
1116
1117         for output in l:
1118             label = output["label"]
1119             revisions = {}
1120             newestChange = 0
1121             if self.verbose:
1122                 print "Querying files for label %s" % label
1123             for file in p4CmdList("files "
1124                                   +  ' '.join (["%s...@%s" % (p, label)
1125                                                 for p in self.depotPaths])):
1126                 revisions[file["depotFile"]] = file["rev"]
1127                 change = int(file["change"])
1128                 if change > newestChange:
1129                     newestChange = change
1130
1131             self.labels[newestChange] = [output, revisions]
1132
1133         if self.verbose:
1134             print "Label changes: %s" % self.labels.keys()
1135
1136     def guessProjectName(self):
1137         for p in self.depotPaths:
1138             if p.endswith("/"):
1139                 p = p[:-1]
1140             p = p[p.strip().rfind("/") + 1:]
1141             if not p.endswith("/"):
1142                p += "/"
1143             return p
1144
1145     def getBranchMapping(self):
1146         lostAndFoundBranches = set()
1147
1148         for info in p4CmdList("branches"):
1149             details = p4Cmd("branch -o %s" % info["branch"])
1150             viewIdx = 0
1151             while details.has_key("View%s" % viewIdx):
1152                 paths = details["View%s" % viewIdx].split(" ")
1153                 viewIdx = viewIdx + 1
1154                 # require standard //depot/foo/... //depot/bar/... mapping
1155                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1156                     continue
1157                 source = paths[0]
1158                 destination = paths[1]
1159                 ## HACK
1160                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1161                     source = source[len(self.depotPaths[0]):-4]
1162                     destination = destination[len(self.depotPaths[0]):-4]
1163
1164                     if destination in self.knownBranches:
1165                         if not self.silent:
1166                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1167                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1168                         continue
1169
1170                     self.knownBranches[destination] = source
1171
1172                     lostAndFoundBranches.discard(destination)
1173
1174                     if source not in self.knownBranches:
1175                         lostAndFoundBranches.add(source)
1176
1177
1178         for branch in lostAndFoundBranches:
1179             self.knownBranches[branch] = branch
1180
1181     def getBranchMappingFromGitBranches(self):
1182         branches = p4BranchesInGit(self.importIntoRemotes)
1183         for branch in branches.keys():
1184             if branch == "master":
1185                 branch = "main"
1186             else:
1187                 branch = branch[len(self.projectName):]
1188             self.knownBranches[branch] = branch
1189
1190     def listExistingP4GitBranches(self):
1191         # branches holds mapping from name to commit
1192         branches = p4BranchesInGit(self.importIntoRemotes)
1193         self.p4BranchesInGit = branches.keys()
1194         for branch in branches.keys():
1195             self.initialParents[self.refPrefix + branch] = branches[branch]
1196
1197     def updateOptionDict(self, d):
1198         option_keys = {}
1199         if self.keepRepoPath:
1200             option_keys['keepRepoPath'] = 1
1201
1202         d["options"] = ' '.join(sorted(option_keys.keys()))
1203
1204     def readOptions(self, d):
1205         self.keepRepoPath = (d.has_key('options')
1206                              and ('keepRepoPath' in d['options']))
1207
1208     def gitRefForBranch(self, branch):
1209         if branch == "main":
1210             return self.refPrefix + "master"
1211
1212         if len(branch) <= 0:
1213             return branch
1214
1215         return self.refPrefix + self.projectName + branch
1216
1217     def gitCommitByP4Change(self, ref, change):
1218         if self.verbose:
1219             print "looking in ref " + ref + " for change %s using bisect..." % change
1220
1221         earliestCommit = ""
1222         latestCommit = parseRevision(ref)
1223
1224         while True:
1225             if self.verbose:
1226                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1227             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1228             if len(next) == 0:
1229                 if self.verbose:
1230                     print "argh"
1231                 return ""
1232             log = extractLogMessageFromGitCommit(next)
1233             settings = extractSettingsGitLog(log)
1234             currentChange = int(settings['change'])
1235             if self.verbose:
1236                 print "current change %s" % currentChange
1237
1238             if currentChange == change:
1239                 if self.verbose:
1240                     print "found %s" % next
1241                 return next
1242
1243             if currentChange < change:
1244                 earliestCommit = "^%s" % next
1245             else:
1246                 latestCommit = "%s" % next
1247
1248         return ""
1249
1250     def importNewBranch(self, branch, maxChange):
1251         # make fast-import flush all changes to disk and update the refs using the checkpoint
1252         # command so that we can try to find the branch parent in the git history
1253         self.gitStream.write("checkpoint\n\n");
1254         self.gitStream.flush();
1255         branchPrefix = self.depotPaths[0] + branch + "/"
1256         range = "@1,%s" % maxChange
1257         #print "prefix" + branchPrefix
1258         changes = p4ChangesForPaths([branchPrefix], range)
1259         if len(changes) <= 0:
1260             return False
1261         firstChange = changes[0]
1262         #print "first change in branch: %s" % firstChange
1263         sourceBranch = self.knownBranches[branch]
1264         sourceDepotPath = self.depotPaths[0] + sourceBranch
1265         sourceRef = self.gitRefForBranch(sourceBranch)
1266         #print "source " + sourceBranch
1267
1268         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1269         #print "branch parent: %s" % branchParentChange
1270         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1271         if len(gitParent) > 0:
1272             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1273             #print "parent git commit: %s" % gitParent
1274
1275         self.importChanges(changes)
1276         return True
1277
1278     def importChanges(self, changes):
1279         cnt = 1
1280         for change in changes:
1281             description = p4Cmd("describe %s" % change)
1282             self.updateOptionDict(description)
1283
1284             if not self.silent:
1285                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1286                 sys.stdout.flush()
1287             cnt = cnt + 1
1288
1289             try:
1290                 if self.detectBranches:
1291                     branches = self.splitFilesIntoBranches(description)
1292                     for branch in branches.keys():
1293                         ## HACK  --hwn
1294                         branchPrefix = self.depotPaths[0] + branch + "/"
1295
1296                         parent = ""
1297
1298                         filesForCommit = branches[branch]
1299
1300                         if self.verbose:
1301                             print "branch is %s" % branch
1302
1303                         self.updatedBranches.add(branch)
1304
1305                         if branch not in self.createdBranches:
1306                             self.createdBranches.add(branch)
1307                             parent = self.knownBranches[branch]
1308                             if parent == branch:
1309                                 parent = ""
1310                             else:
1311                                 fullBranch = self.projectName + branch
1312                                 if fullBranch not in self.p4BranchesInGit:
1313                                     if not self.silent:
1314                                         print("\n    Importing new branch %s" % fullBranch);
1315                                     if self.importNewBranch(branch, change - 1):
1316                                         parent = ""
1317                                         self.p4BranchesInGit.append(fullBranch)
1318                                     if not self.silent:
1319                                         print("\n    Resuming with change %s" % change);
1320
1321                                 if self.verbose:
1322                                     print "parent determined through known branches: %s" % parent
1323
1324                         branch = self.gitRefForBranch(branch)
1325                         parent = self.gitRefForBranch(parent)
1326
1327                         if self.verbose:
1328                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1329
1330                         if len(parent) == 0 and branch in self.initialParents:
1331                             parent = self.initialParents[branch]
1332                             del self.initialParents[branch]
1333
1334                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1335                 else:
1336                     files = self.extractFilesFromCommit(description)
1337                     self.commit(description, files, self.branch, self.depotPaths,
1338                                 self.initialParent)
1339                     self.initialParent = ""
1340             except IOError:
1341                 print self.gitError.read()
1342                 sys.exit(1)
1343
1344     def importHeadRevision(self, revision):
1345         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1346
1347         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1348         details["desc"] = ("Initial import of %s from the state at revision %s"
1349                            % (' '.join(self.depotPaths), revision))
1350         details["change"] = revision
1351         newestRevision = 0
1352
1353         fileCnt = 0
1354         for info in p4CmdList("files "
1355                               +  ' '.join(["%s...%s"
1356                                            % (p, revision)
1357                                            for p in self.depotPaths])):
1358
1359             if info['code'] == 'error':
1360                 sys.stderr.write("p4 returned an error: %s\n"
1361                                  % info['data'])
1362                 sys.exit(1)
1363
1364
1365             change = int(info["change"])
1366             if change > newestRevision:
1367                 newestRevision = change
1368
1369             if info["action"] == "delete":
1370                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1371                 #fileCnt = fileCnt + 1
1372                 continue
1373
1374             for prop in ["depotFile", "rev", "action", "type" ]:
1375                 details["%s%s" % (prop, fileCnt)] = info[prop]
1376
1377             fileCnt = fileCnt + 1
1378
1379         details["change"] = newestRevision
1380         self.updateOptionDict(details)
1381         try:
1382             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1383         except IOError:
1384             print "IO error with git fast-import. Is your git version recent enough?"
1385             print self.gitError.read()
1386
1387
1388     def run(self, args):
1389         self.depotPaths = []
1390         self.changeRange = ""
1391         self.initialParent = ""
1392         self.previousDepotPaths = []
1393
1394         # map from branch depot path to parent branch
1395         self.knownBranches = {}
1396         self.initialParents = {}
1397         self.hasOrigin = originP4BranchesExist()
1398         if not self.syncWithOrigin:
1399             self.hasOrigin = False
1400
1401         if self.importIntoRemotes:
1402             self.refPrefix = "refs/remotes/p4/"
1403         else:
1404             self.refPrefix = "refs/heads/p4/"
1405
1406         if self.syncWithOrigin and self.hasOrigin:
1407             if not self.silent:
1408                 print "Syncing with origin first by calling git fetch origin"
1409             system("git fetch origin")
1410
1411         if len(self.branch) == 0:
1412             self.branch = self.refPrefix + "master"
1413             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1414                 system("git update-ref %s refs/heads/p4" % self.branch)
1415                 system("git branch -D p4");
1416             # create it /after/ importing, when master exists
1417             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1418                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1419
1420         # TODO: should always look at previous commits,
1421         # merge with previous imports, if possible.
1422         if args == []:
1423             if self.hasOrigin:
1424                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1425             self.listExistingP4GitBranches()
1426
1427             if len(self.p4BranchesInGit) > 1:
1428                 if not self.silent:
1429                     print "Importing from/into multiple branches"
1430                 self.detectBranches = True
1431
1432             if self.verbose:
1433                 print "branches: %s" % self.p4BranchesInGit
1434
1435             p4Change = 0
1436             for branch in self.p4BranchesInGit:
1437                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1438
1439                 settings = extractSettingsGitLog(logMsg)
1440
1441                 self.readOptions(settings)
1442                 if (settings.has_key('depot-paths')
1443                     and settings.has_key ('change')):
1444                     change = int(settings['change']) + 1
1445                     p4Change = max(p4Change, change)
1446
1447                     depotPaths = sorted(settings['depot-paths'])
1448                     if self.previousDepotPaths == []:
1449                         self.previousDepotPaths = depotPaths
1450                     else:
1451                         paths = []
1452                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1453                             for i in range(0, min(len(cur), len(prev))):
1454                                 if cur[i] <> prev[i]:
1455                                     i = i - 1
1456                                     break
1457
1458                             paths.append (cur[:i + 1])
1459
1460                         self.previousDepotPaths = paths
1461
1462             if p4Change > 0:
1463                 self.depotPaths = sorted(self.previousDepotPaths)
1464                 self.changeRange = "@%s,#head" % p4Change
1465                 if not self.detectBranches:
1466                     self.initialParent = parseRevision(self.branch)
1467                 if not self.silent and not self.detectBranches:
1468                     print "Performing incremental import into %s git branch" % self.branch
1469
1470         if not self.branch.startswith("refs/"):
1471             self.branch = "refs/heads/" + self.branch
1472
1473         if len(args) == 0 and self.depotPaths:
1474             if not self.silent:
1475                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1476         else:
1477             if self.depotPaths and self.depotPaths != args:
1478                 print ("previous import used depot path %s and now %s was specified. "
1479                        "This doesn't work!" % (' '.join (self.depotPaths),
1480                                                ' '.join (args)))
1481                 sys.exit(1)
1482
1483             self.depotPaths = sorted(args)
1484
1485         revision = ""
1486         self.users = {}
1487
1488         newPaths = []
1489         for p in self.depotPaths:
1490             if p.find("@") != -1:
1491                 atIdx = p.index("@")
1492                 self.changeRange = p[atIdx:]
1493                 if self.changeRange == "@all":
1494                     self.changeRange = ""
1495                 elif ',' not in self.changeRange:
1496                     revision = self.changeRange
1497                     self.changeRange = ""
1498                 p = p[:atIdx]
1499             elif p.find("#") != -1:
1500                 hashIdx = p.index("#")
1501                 revision = p[hashIdx:]
1502                 p = p[:hashIdx]
1503             elif self.previousDepotPaths == []:
1504                 revision = "#head"
1505
1506             p = re.sub ("\.\.\.$", "", p)
1507             if not p.endswith("/"):
1508                 p += "/"
1509
1510             newPaths.append(p)
1511
1512         self.depotPaths = newPaths
1513
1514
1515         self.loadUserMapFromCache()
1516         self.labels = {}
1517         if self.detectLabels:
1518             self.getLabels();
1519
1520         if self.detectBranches:
1521             ## FIXME - what's a P4 projectName ?
1522             self.projectName = self.guessProjectName()
1523
1524             if self.hasOrigin:
1525                 self.getBranchMappingFromGitBranches()
1526             else:
1527                 self.getBranchMapping()
1528             if self.verbose:
1529                 print "p4-git branches: %s" % self.p4BranchesInGit
1530                 print "initial parents: %s" % self.initialParents
1531             for b in self.p4BranchesInGit:
1532                 if b != "master":
1533
1534                     ## FIXME
1535                     b = b[len(self.projectName):]
1536                 self.createdBranches.add(b)
1537
1538         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1539
1540         importProcess = subprocess.Popen(["git", "fast-import"],
1541                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1542                                          stderr=subprocess.PIPE);
1543         self.gitOutput = importProcess.stdout
1544         self.gitStream = importProcess.stdin
1545         self.gitError = importProcess.stderr
1546
1547         if revision:
1548             self.importHeadRevision(revision)
1549         else:
1550             changes = []
1551
1552             if len(self.changesFile) > 0:
1553                 output = open(self.changesFile).readlines()
1554                 changeSet = Set()
1555                 for line in output:
1556                     changeSet.add(int(line))
1557
1558                 for change in changeSet:
1559                     changes.append(change)
1560
1561                 changes.sort()
1562             else:
1563                 if self.verbose:
1564                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1565                                                               self.changeRange)
1566                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1567
1568                 if len(self.maxChanges) > 0:
1569                     changes = changes[:min(int(self.maxChanges), len(changes))]
1570
1571             if len(changes) == 0:
1572                 if not self.silent:
1573                     print "No changes to import!"
1574                 return True
1575
1576             if not self.silent and not self.detectBranches:
1577                 print "Import destination: %s" % self.branch
1578
1579             self.updatedBranches = set()
1580
1581             self.importChanges(changes)
1582
1583             if not self.silent:
1584                 print ""
1585                 if len(self.updatedBranches) > 0:
1586                     sys.stdout.write("Updated branches: ")
1587                     for b in self.updatedBranches:
1588                         sys.stdout.write("%s " % b)
1589                     sys.stdout.write("\n")
1590
1591         self.gitStream.close()
1592         if importProcess.wait() != 0:
1593             die("fast-import failed: %s" % self.gitError.read())
1594         self.gitOutput.close()
1595         self.gitError.close()
1596
1597         return True
1598
1599 class P4Rebase(Command):
1600     def __init__(self):
1601         Command.__init__(self)
1602         self.options = [ ]
1603         self.description = ("Fetches the latest revision from perforce and "
1604                             + "rebases the current work (branch) against it")
1605         self.verbose = False
1606
1607     def run(self, args):
1608         sync = P4Sync()
1609         sync.run([])
1610
1611         return self.rebase()
1612
1613     def rebase(self):
1614         if os.system("git update-index --refresh") != 0:
1615             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1616         if len(read_pipe("git diff-index HEAD --")) > 0:
1617             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1618
1619         [upstream, settings] = findUpstreamBranchPoint()
1620         if len(upstream) == 0:
1621             die("Cannot find upstream branchpoint for rebase")
1622
1623         # the branchpoint may be p4/foo~3, so strip off the parent
1624         upstream = re.sub("~[0-9]+$", "", upstream)
1625
1626         print "Rebasing the current branch onto %s" % upstream
1627         oldHead = read_pipe("git rev-parse HEAD").strip()
1628         system("git rebase %s" % upstream)
1629         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1630         return True
1631
1632 class P4Clone(P4Sync):
1633     def __init__(self):
1634         P4Sync.__init__(self)
1635         self.description = "Creates a new git repository and imports from Perforce into it"
1636         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1637         self.options += [
1638             optparse.make_option("--destination", dest="cloneDestination",
1639                                  action='store', default=None,
1640                                  help="where to leave result of the clone"),
1641             optparse.make_option("-/", dest="cloneExclude",
1642                                  action="append", type="string",
1643                                  help="exclude depot path")
1644         ]
1645         self.cloneDestination = None
1646         self.needsGit = False
1647
1648     # This is required for the "append" cloneExclude action
1649     def ensure_value(self, attr, value):
1650         if not hasattr(self, attr) or getattr(self, attr) is None:
1651             setattr(self, attr, value)
1652         return getattr(self, attr)
1653
1654     def defaultDestination(self, args):
1655         ## TODO: use common prefix of args?
1656         depotPath = args[0]
1657         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1658         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1659         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1660         depotDir = re.sub(r"/$", "", depotDir)
1661         return os.path.split(depotDir)[1]
1662
1663     def run(self, args):
1664         if len(args) < 1:
1665             return False
1666
1667         if self.keepRepoPath and not self.cloneDestination:
1668             sys.stderr.write("Must specify destination for --keep-path\n")
1669             sys.exit(1)
1670
1671         depotPaths = args
1672
1673         if not self.cloneDestination and len(depotPaths) > 1:
1674             self.cloneDestination = depotPaths[-1]
1675             depotPaths = depotPaths[:-1]
1676
1677         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1678         for p in depotPaths:
1679             if not p.startswith("//"):
1680                 return False
1681
1682         if not self.cloneDestination:
1683             self.cloneDestination = self.defaultDestination(args)
1684
1685         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1686         if not os.path.exists(self.cloneDestination):
1687             os.makedirs(self.cloneDestination)
1688         os.chdir(self.cloneDestination)
1689         system("git init")
1690         self.gitdir = os.getcwd() + "/.git"
1691         if not P4Sync.run(self, depotPaths):
1692             return False
1693         if self.branch != "master":
1694             if gitBranchExists("refs/remotes/p4/master"):
1695                 system("git branch master refs/remotes/p4/master")
1696                 system("git checkout -f")
1697             else:
1698                 print "Could not detect main branch. No checkout/master branch created."
1699
1700         return True
1701
1702 class P4Branches(Command):
1703     def __init__(self):
1704         Command.__init__(self)
1705         self.options = [ ]
1706         self.description = ("Shows the git branches that hold imports and their "
1707                             + "corresponding perforce depot paths")
1708         self.verbose = False
1709
1710     def run(self, args):
1711         if originP4BranchesExist():
1712             createOrUpdateBranchesFromOrigin()
1713
1714         cmdline = "git rev-parse --symbolic "
1715         cmdline += " --remotes"
1716
1717         for line in read_pipe_lines(cmdline):
1718             line = line.strip()
1719
1720             if not line.startswith('p4/') or line == "p4/HEAD":
1721                 continue
1722             branch = line
1723
1724             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1725             settings = extractSettingsGitLog(log)
1726
1727             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1728         return True
1729
1730 class HelpFormatter(optparse.IndentedHelpFormatter):
1731     def __init__(self):
1732         optparse.IndentedHelpFormatter.__init__(self)
1733
1734     def format_description(self, description):
1735         if description:
1736             return description + "\n"
1737         else:
1738             return ""
1739
1740 def printUsage(commands):
1741     print "usage: %s <command> [options]" % sys.argv[0]
1742     print ""
1743     print "valid commands: %s" % ", ".join(commands)
1744     print ""
1745     print "Try %s <command> --help for command specific help." % sys.argv[0]
1746     print ""
1747
1748 commands = {
1749     "debug" : P4Debug,
1750     "submit" : P4Submit,
1751     "commit" : P4Submit,
1752     "sync" : P4Sync,
1753     "rebase" : P4Rebase,
1754     "clone" : P4Clone,
1755     "rollback" : P4RollBack,
1756     "branches" : P4Branches
1757 }
1758
1759
1760 def main():
1761     if len(sys.argv[1:]) == 0:
1762         printUsage(commands.keys())
1763         sys.exit(2)
1764
1765     cmd = ""
1766     cmdName = sys.argv[1]
1767     try:
1768         klass = commands[cmdName]
1769         cmd = klass()
1770     except KeyError:
1771         print "unknown command %s" % cmdName
1772         print ""
1773         printUsage(commands.keys())
1774         sys.exit(2)
1775
1776     options = cmd.options
1777     cmd.gitdir = os.environ.get("GIT_DIR", None)
1778
1779     args = sys.argv[2:]
1780
1781     if len(options) > 0:
1782         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1783
1784         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1785                                        options,
1786                                        description = cmd.description,
1787                                        formatter = HelpFormatter())
1788
1789         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1790     global verbose
1791     verbose = cmd.verbose
1792     if cmd.needsGit:
1793         if cmd.gitdir == None:
1794             cmd.gitdir = os.path.abspath(".git")
1795             if not isValidGitDir(cmd.gitdir):
1796                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1797                 if os.path.exists(cmd.gitdir):
1798                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1799                     if len(cdup) > 0:
1800                         os.chdir(cdup);
1801
1802         if not isValidGitDir(cmd.gitdir):
1803             if isValidGitDir(cmd.gitdir + "/.git"):
1804                 cmd.gitdir += "/.git"
1805             else:
1806                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1807
1808         os.environ["GIT_DIR"] = cmd.gitdir
1809
1810     if not cmd.run(args):
1811         parser.print_help()
1812
1813
1814 if __name__ == '__main__':
1815     main()