contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)\r?$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s (result: '%s')" % (file, result))
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                 optparse.make_option("--origin", dest="origin"),
 469                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 470         ]
 471         self.description = "Submit changes from git to the perforce depot."
 472         self.usage += " [name of git branch to submit into perforce depot]"
 473         self.interactive = True
 474         self.origin = ""
 475         self.detectRename = False
 476         self.verbose = False
 477         self.isWindows = (platform.system() == "Windows")
 478
 479     def check(self):
 480         if len(p4CmdList("opened ...")) > 0:
 481             die("You have files opened with perforce! Close them before starting the sync.")
 482
 483     # replaces everything between 'Description:' and the next P4 submit template field with the
 484     # commit message
 485     def prepareLogMessage(self, template, message):
 486         result = ""
 487
 488         inDescriptionSection = False
 489
 490         for line in template.split("\n"):
 491             if line.startswith("#"):
 492                 result += line + "\n"
 493                 continue
 494
 495             if inDescriptionSection:
 496                 if line.startswith("Files:"):
 497                     inDescriptionSection = False
 498                 else:
 499                     continue
 500             else:
 501                 if line.startswith("Description:"):
 502                     inDescriptionSection = True
 503                     line += "\n"
 504                     for messageLine in message.split("\n"):
 505                         line += "\t" + messageLine + "\n"
 506
 507             result += line + "\n"
 508
 509         return result
 510
 511     def prepareSubmitTemplate(self):
 512         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513         template = ""
 514         inFilesSection = False
 515         for line in read_pipe_lines("p4 change -o"):
 516             if line.endswith("\r\n"):
 517                 line = line[:-2] + "\n"
 518             if inFilesSection:
 519                 if line.startswith("\t"):
 520                     # path starts and ends with a tab
 521                     path = line[1:]
 522                     lastTab = path.rfind("\t")
 523                     if lastTab != -1:
 524                         path = path[:lastTab]
 525                         if not path.startswith(self.depotPath):
 526                             continue
 527                 else:
 528                     inFilesSection = False
 529             else:
 530                 if line.startswith("Files:"):
 531                     inFilesSection = True
 532
 533             template += line
 534
 535         return template
 536
 537     def applyCommit(self, id):
 538         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 539         diffOpts = ("", "-M")[self.detectRename]
 540         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 541         filesToAdd = set()
 542         filesToDelete = set()
 543         editedFiles = set()
 544         filesToChangeExecBit = {}
 545         for line in diff:
 546             diff = parseDiffTreeEntry(line)
 547             modifier = diff['status']
 548             path = diff['src']
 549             if modifier == "M":
 550                 system("p4 edit \"%s\"" % path)
 551                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 552                     filesToChangeExecBit[path] = diff['dst_mode']
 553                 editedFiles.add(path)
 554             elif modifier == "A":
 555                 filesToAdd.add(path)
 556                 filesToChangeExecBit[path] = diff['dst_mode']
 557                 if path in filesToDelete:
 558                     filesToDelete.remove(path)
 559             elif modifier == "D":
 560                 filesToDelete.add(path)
 561                 if path in filesToAdd:
 562                     filesToAdd.remove(path)
 563             elif modifier == "R":
 564                 src, dest = diff['src'], diff['dst']
 565                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 566                 system("p4 edit \"%s\"" % (dest))
 567                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 568                     filesToChangeExecBit[dest] = diff['dst_mode']
 569                 os.unlink(dest)
 570                 editedFiles.add(dest)
 571                 filesToDelete.add(src)
 572             else:
 573                 die("unknown modifier %s for %s" % (modifier, path))
 574
 575         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 576         patchcmd = diffcmd + " | git apply "
 577         tryPatchCmd = patchcmd + "--check -"
 578         applyPatchCmd = patchcmd + "--check --apply -"
 579
 580         if os.system(tryPatchCmd) != 0:
 581             print "Unfortunately applying the change failed!"
 582             print "What do you want to do?"
 583             response = "x"
 584             while response != "s" and response != "a" and response != "w":
 585                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 586                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 587             if response == "s":
 588                 print "Skipping! Good luck with the next patches..."
 589                 for f in editedFiles:
 590                     system("p4 revert \"%s\"" % f);
 591                 for f in filesToAdd:
 592                     system("rm %s" %f)
 593                 return
 594             elif response == "a":
 595                 os.system(applyPatchCmd)
 596                 if len(filesToAdd) > 0:
 597                     print "You may also want to call p4 add on the following files:"
 598                     print " ".join(filesToAdd)
 599                 if len(filesToDelete):
 600                     print "The following files should be scheduled for deletion with p4 delete:"
 601                     print " ".join(filesToDelete)
 602                 die("Please resolve and submit the conflict manually and "
 603                     + "continue afterwards with git-p4 submit --continue")
 604             elif response == "w":
 605                 system(diffcmd + " > patch.txt")
 606                 print "Patch saved to patch.txt in %s !" % self.clientPath
 607                 die("Please resolve and submit the conflict manually and "
 608                     "continue afterwards with git-p4 submit --continue")
 609
 610         system(applyPatchCmd)
 611
 612         for f in filesToAdd:
 613             system("p4 add \"%s\"" % f)
 614         for f in filesToDelete:
 615             system("p4 revert \"%s\"" % f)
 616             system("p4 delete \"%s\"" % f)
 617
 618         # Set/clear executable bits
 619         for f in filesToChangeExecBit.keys():
 620             mode = filesToChangeExecBit[f]
 621             setP4ExecBit(f, mode)
 622
 623         logMessage = extractLogMessageFromGitCommit(id)
 624         logMessage = logMessage.strip()
 625
 626         template = self.prepareSubmitTemplate()
 627
 628         if self.interactive:
 629             submitTemplate = self.prepareLogMessage(template, logMessage)
 630             if os.environ.has_key("P4DIFF"):
 631                 del(os.environ["P4DIFF"])
 632             diff = read_pipe("p4 diff -du ...")
 633
 634             newdiff = ""
 635             for newFile in filesToAdd:
 636                 newdiff += "==== new file ====\n"
 637                 newdiff += "--- /dev/null\n"
 638                 newdiff += "+++ %s\n" % newFile
 639                 f = open(newFile, "r")
 640                 for line in f.readlines():
 641                     newdiff += "+" + line
 642                 f.close()
 643
 644             separatorLine = "######## everything below this line is just the diff #######\n"
 645
 646             [handle, fileName] = tempfile.mkstemp()
 647             tmpFile = os.fdopen(handle, "w+")
 648             if self.isWindows:
 649                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 650                 separatorLine = separatorLine.replace("\n", "\r\n")
 651                 newdiff = newdiff.replace("\n", "\r\n")
 652             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 653             tmpFile.close()
 654             defaultEditor = "vi"
 655             if platform.system() == "Windows":
 656                 defaultEditor = "notepad"
 657             if os.environ.has_key("P4EDITOR"):
 658                 editor = os.environ.get("P4EDITOR")
 659             else:
 660                 editor = os.environ.get("EDITOR", defaultEditor);
 661             system(editor + " " + fileName)
 662             tmpFile = open(fileName, "rb")
 663             message = tmpFile.read()
 664             tmpFile.close()
 665             os.remove(fileName)
 666             submitTemplate = message[:message.index(separatorLine)]
 667             if self.isWindows:
 668                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 669
 670             write_pipe("p4 submit -i", submitTemplate)
 671         else:
 672             fileName = "submit.txt"
 673             file = open(fileName, "w+")
 674             file.write(self.prepareLogMessage(template, logMessage))
 675             file.close()
 676             print ("Perforce submit template written as %s. "
 677                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 678                    % (fileName, fileName))
 679
 680     def run(self, args):
 681         if len(args) == 0:
 682             self.master = currentGitBranch()
 683             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 684                 die("Detecting current git branch failed!")
 685         elif len(args) == 1:
 686             self.master = args[0]
 687         else:
 688             return False
 689
 690         allowSubmit = gitConfig("git-p4.allowSubmit")
 691         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 692             die("%s is not in git-p4.allowSubmit" % self.master)
 693
 694         [upstream, settings] = findUpstreamBranchPoint()
 695         self.depotPath = settings['depot-paths'][0]
 696         if len(self.origin) == 0:
 697             self.origin = upstream
 698
 699         if self.verbose:
 700             print "Origin branch is " + self.origin
 701
 702         if len(self.depotPath) == 0:
 703             print "Internal error: cannot locate perforce depot path from existing branches"
 704             sys.exit(128)
 705
 706         self.clientPath = p4Where(self.depotPath)
 707
 708         if len(self.clientPath) == 0:
 709             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 710             sys.exit(128)
 711
 712         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 713         self.oldWorkingDirectory = os.getcwd()
 714
 715         os.chdir(self.clientPath)
 716         print "Syncronizing p4 checkout..."
 717         system("p4 sync ...")
 718
 719         self.check()
 720
 721         commits = []
 722         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 723             commits.append(line.strip())
 724         commits.reverse()
 725
 726         while len(commits) > 0:
 727             commit = commits[0]
 728             commits = commits[1:]
 729             self.applyCommit(commit)
 730             if not self.interactive:
 731                 break
 732
 733         if len(commits) == 0:
 734             print "All changes applied!"
 735             os.chdir(self.oldWorkingDirectory)
 736
 737             sync = P4Sync()
 738             sync.run([])
 739
 740             rebase = P4Rebase()
 741             rebase.rebase()
 742
 743         return True
 744
 745 class P4Sync(Command):
 746     def __init__(self):
 747         Command.__init__(self)
 748         self.options = [
 749                 optparse.make_option("--branch", dest="branch"),
 750                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 751                 optparse.make_option("--changesfile", dest="changesFile"),
 752                 optparse.make_option("--silent", dest="silent", action="store_true"),
 753                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 754                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 755                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 756                                      help="Import into refs/heads/ , not refs/remotes"),
 757                 optparse.make_option("--max-changes", dest="maxChanges"),
 758                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 759                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 760                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 761                                      help="Only sync files that are included in the Perforce Client Spec")
 762         ]
 763         self.description = """Imports from Perforce into a git repository.\n
 764     example:
 765     //depot/my/project/ -- to import the current head
 766     //depot/my/project/@all -- to import everything
 767     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 768
 769     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 770
 771         self.usage += " //depot/path[@revRange]"
 772         self.silent = False
 773         self.createdBranches = Set()
 774         self.committedChanges = Set()
 775         self.branch = ""
 776         self.detectBranches = False
 777         self.detectLabels = False
 778         self.changesFile = ""
 779         self.syncWithOrigin = True
 780         self.verbose = False
 781         self.importIntoRemotes = True
 782         self.maxChanges = ""
 783         self.isWindows = (platform.system() == "Windows")
 784         self.keepRepoPath = False
 785         self.depotPaths = None
 786         self.p4BranchesInGit = []
 787         self.cloneExclude = []
 788         self.useClientSpec = False
 789         self.clientSpecDirs = []
 790
 791         if gitConfig("git-p4.syncFromOrigin") == "false":
 792             self.syncWithOrigin = False
 793
 794     def extractFilesFromCommit(self, commit):
 795         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 796                              for path in self.cloneExclude]
 797         files = []
 798         fnum = 0
 799         while commit.has_key("depotFile%s" % fnum):
 800             path =  commit["depotFile%s" % fnum]
 801
 802             if [p for p in self.cloneExclude
 803                 if path.startswith (p)]:
 804                 found = False
 805             else:
 806                 found = [p for p in self.depotPaths
 807                          if path.startswith (p)]
 808             if not found:
 809                 fnum = fnum + 1
 810                 continue
 811
 812             file = {}
 813             file["path"] = path
 814             file["rev"] = commit["rev%s" % fnum]
 815             file["action"] = commit["action%s" % fnum]
 816             file["type"] = commit["type%s" % fnum]
 817             files.append(file)
 818             fnum = fnum + 1
 819         return files
 820
 821     def stripRepoPath(self, path, prefixes):
 822         if self.keepRepoPath:
 823             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 824
 825         for p in prefixes:
 826             if path.startswith(p):
 827                 path = path[len(p):]
 828
 829         return path
 830
 831     def splitFilesIntoBranches(self, commit):
 832         branches = {}
 833         fnum = 0
 834         while commit.has_key("depotFile%s" % fnum):
 835             path =  commit["depotFile%s" % fnum]
 836             found = [p for p in self.depotPaths
 837                      if path.startswith (p)]
 838             if not found:
 839                 fnum = fnum + 1
 840                 continue
 841
 842             file = {}
 843             file["path"] = path
 844             file["rev"] = commit["rev%s" % fnum]
 845             file["action"] = commit["action%s" % fnum]
 846             file["type"] = commit["type%s" % fnum]
 847             fnum = fnum + 1
 848
 849             relPath = self.stripRepoPath(path, self.depotPaths)
 850
 851             for branch in self.knownBranches.keys():
 852
 853                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 854                 if relPath.startswith(branch + "/"):
 855                     if branch not in branches:
 856                         branches[branch] = []
 857                     branches[branch].append(file)
 858                     break
 859
 860         return branches
 861
 862     ## Should move this out, doesn't use SELF.
 863     def readP4Files(self, files):
 864         filesForCommit = []
 865         filesToRead = []
 866
 867         for f in files:
 868             includeFile = True
 869             for val in self.clientSpecDirs:
 870                 if f['path'].startswith(val[0]):
 871                     if val[1] <= 0:
 872                         includeFile = False
 873                     break
 874
 875             if includeFile:
 876                 filesForCommit.append(f)
 877                 if f['action'] != 'delete':
 878                     filesToRead.append(f)
 879
 880         filedata = []
 881         if len(filesToRead) > 0:
 882             filedata = p4CmdList('-x - print',
 883                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 884                                                   for f in filesToRead]),
 885                                  stdin_mode='w+')
 886
 887             if "p4ExitCode" in filedata[0]:
 888                 die("Problems executing p4. Error: [%d]."
 889                     % (filedata[0]['p4ExitCode']));
 890
 891         j = 0;
 892         contents = {}
 893         while j < len(filedata):
 894             stat = filedata[j]
 895             j += 1
 896             text = [];
 897             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 898                 text.append(filedata[j]['data'])
 899                 j += 1
 900             text = ''.join(text)
 901
 902             if not stat.has_key('depotFile'):
 903                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 904                 continue
 905
 906             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 907                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 908             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 909                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 910
 911             contents[stat['depotFile']] = text
 912
 913         for f in filesForCommit:
 914             path = f['path']
 915             if contents.has_key(path):
 916                 f['data'] = contents[path]
 917
 918         return filesForCommit
 919
 920     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 921         epoch = details["time"]
 922         author = details["user"]
 923
 924         if self.verbose:
 925             print "commit into %s" % branch
 926
 927         # start with reading files; if that fails, we should not
 928         # create a commit.
 929         new_files = []
 930         for f in files:
 931             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 932                 new_files.append (f)
 933             else:
 934                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 935         files = self.readP4Files(new_files)
 936
 937         self.gitStream.write("commit %s\n" % branch)
 938 #        gitStream.write("mark :%s\n" % details["change"])
 939         self.committedChanges.add(int(details["change"]))
 940         committer = ""
 941         if author not in self.users:
 942             self.getUserMapFromPerforceServer()
 943         if author in self.users:
 944             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 945         else:
 946             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 947
 948         self.gitStream.write("committer %s\n" % committer)
 949
 950         self.gitStream.write("data <<EOT\n")
 951         self.gitStream.write(details["desc"])
 952         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 953                              % (','.join (branchPrefixes), details["change"]))
 954         if len(details['options']) > 0:
 955             self.gitStream.write(": options = %s" % details['options'])
 956         self.gitStream.write("]\nEOT\n\n")
 957
 958         if len(parent) > 0:
 959             if self.verbose:
 960                 print "parent %s" % parent
 961             self.gitStream.write("from %s\n" % parent)
 962
 963         for file in files:
 964             if file["type"] == "apple":
 965                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 966                 continue
 967
 968             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 969             if file["action"] == "delete":
 970                 self.gitStream.write("D %s\n" % relPath)
 971             else:
 972                 data = file['data']
 973
 974                 mode = "644"
 975                 if isP4Exec(file["type"]):
 976                     mode = "755"
 977                 elif file["type"] == "symlink":
 978                     mode = "120000"
 979                     # p4 print on a symlink contains "target\n", so strip it off
 980                     data = data[:-1]
 981
 982                 if self.isWindows and file["type"].endswith("text"):
 983                     data = data.replace("\r\n", "\n")
 984
 985                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 986                 self.gitStream.write("data %s\n" % len(data))
 987                 self.gitStream.write(data)
 988                 self.gitStream.write("\n")
 989
 990         self.gitStream.write("\n")
 991
 992         change = int(details["change"])
 993
 994         if self.labels.has_key(change):
 995             label = self.labels[change]
 996             labelDetails = label[0]
 997             labelRevisions = label[1]
 998             if self.verbose:
 999                 print "Change %s is labelled %s" % (change, labelDetails)
1000
1001             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1002                                                     for p in branchPrefixes]))
1003
1004             if len(files) == len(labelRevisions):
1005
1006                 cleanedFiles = {}
1007                 for info in files:
1008                     if info["action"] == "delete":
1009                         continue
1010                     cleanedFiles[info["depotFile"]] = info["rev"]
1011
1012                 if cleanedFiles == labelRevisions:
1013                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1014                     self.gitStream.write("from %s\n" % branch)
1015
1016                     owner = labelDetails["Owner"]
1017                     tagger = ""
1018                     if author in self.users:
1019                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1020                     else:
1021                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1022                     self.gitStream.write("tagger %s\n" % tagger)
1023                     self.gitStream.write("data <<EOT\n")
1024                     self.gitStream.write(labelDetails["Description"])
1025                     self.gitStream.write("EOT\n\n")
1026
1027                 else:
1028                     if not self.silent:
1029                         print ("Tag %s does not match with change %s: files do not match."
1030                                % (labelDetails["label"], change))
1031
1032             else:
1033                 if not self.silent:
1034                     print ("Tag %s does not match with change %s: file count is different."
1035                            % (labelDetails["label"], change))
1036
1037     def getUserCacheFilename(self):
1038         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1039         return home + "/.gitp4-usercache.txt"
1040
1041     def getUserMapFromPerforceServer(self):
1042         if self.userMapFromPerforceServer:
1043             return
1044         self.users = {}
1045
1046         for output in p4CmdList("users"):
1047             if not output.has_key("User"):
1048                 continue
1049             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1050
1051
1052         s = ''
1053         for (key, val) in self.users.items():
1054             s += "%s\t%s\n" % (key, val)
1055
1056         open(self.getUserCacheFilename(), "wb").write(s)
1057         self.userMapFromPerforceServer = True
1058
1059     def loadUserMapFromCache(self):
1060         self.users = {}
1061         self.userMapFromPerforceServer = False
1062         try:
1063             cache = open(self.getUserCacheFilename(), "rb")
1064             lines = cache.readlines()
1065             cache.close()
1066             for line in lines:
1067                 entry = line.strip().split("\t")
1068                 self.users[entry[0]] = entry[1]
1069         except IOError:
1070             self.getUserMapFromPerforceServer()
1071
1072     def getLabels(self):
1073         self.labels = {}
1074
1075         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1076         if len(l) > 0 and not self.silent:
1077             print "Finding files belonging to labels in %s" % `self.depotPaths`
1078
1079         for output in l:
1080             label = output["label"]
1081             revisions = {}
1082             newestChange = 0
1083             if self.verbose:
1084                 print "Querying files for label %s" % label
1085             for file in p4CmdList("files "
1086                                   +  ' '.join (["%s...@%s" % (p, label)
1087                                                 for p in self.depotPaths])):
1088                 revisions[file["depotFile"]] = file["rev"]
1089                 change = int(file["change"])
1090                 if change > newestChange:
1091                     newestChange = change
1092
1093             self.labels[newestChange] = [output, revisions]
1094
1095         if self.verbose:
1096             print "Label changes: %s" % self.labels.keys()
1097
1098     def guessProjectName(self):
1099         for p in self.depotPaths:
1100             if p.endswith("/"):
1101                 p = p[:-1]
1102             p = p[p.strip().rfind("/") + 1:]
1103             if not p.endswith("/"):
1104                p += "/"
1105             return p
1106
1107     def getBranchMapping(self):
1108         lostAndFoundBranches = set()
1109
1110         for info in p4CmdList("branches"):
1111             details = p4Cmd("branch -o %s" % info["branch"])
1112             viewIdx = 0
1113             while details.has_key("View%s" % viewIdx):
1114                 paths = details["View%s" % viewIdx].split(" ")
1115                 viewIdx = viewIdx + 1
1116                 # require standard //depot/foo/... //depot/bar/... mapping
1117                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1118                     continue
1119                 source = paths[0]
1120                 destination = paths[1]
1121                 ## HACK
1122                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1123                     source = source[len(self.depotPaths[0]):-4]
1124                     destination = destination[len(self.depotPaths[0]):-4]
1125
1126                     if destination in self.knownBranches:
1127                         if not self.silent:
1128                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1129                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1130                         continue
1131
1132                     self.knownBranches[destination] = source
1133
1134                     lostAndFoundBranches.discard(destination)
1135
1136                     if source not in self.knownBranches:
1137                         lostAndFoundBranches.add(source)
1138
1139
1140         for branch in lostAndFoundBranches:
1141             self.knownBranches[branch] = branch
1142
1143     def getBranchMappingFromGitBranches(self):
1144         branches = p4BranchesInGit(self.importIntoRemotes)
1145         for branch in branches.keys():
1146             if branch == "master":
1147                 branch = "main"
1148             else:
1149                 branch = branch[len(self.projectName):]
1150             self.knownBranches[branch] = branch
1151
1152     def listExistingP4GitBranches(self):
1153         # branches holds mapping from name to commit
1154         branches = p4BranchesInGit(self.importIntoRemotes)
1155         self.p4BranchesInGit = branches.keys()
1156         for branch in branches.keys():
1157             self.initialParents[self.refPrefix + branch] = branches[branch]
1158
1159     def updateOptionDict(self, d):
1160         option_keys = {}
1161         if self.keepRepoPath:
1162             option_keys['keepRepoPath'] = 1
1163
1164         d["options"] = ' '.join(sorted(option_keys.keys()))
1165
1166     def readOptions(self, d):
1167         self.keepRepoPath = (d.has_key('options')
1168                              and ('keepRepoPath' in d['options']))
1169
1170     def gitRefForBranch(self, branch):
1171         if branch == "main":
1172             return self.refPrefix + "master"
1173
1174         if len(branch) <= 0:
1175             return branch
1176
1177         return self.refPrefix + self.projectName + branch
1178
1179     def gitCommitByP4Change(self, ref, change):
1180         if self.verbose:
1181             print "looking in ref " + ref + " for change %s using bisect..." % change
1182
1183         earliestCommit = ""
1184         latestCommit = parseRevision(ref)
1185
1186         while True:
1187             if self.verbose:
1188                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1189             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1190             if len(next) == 0:
1191                 if self.verbose:
1192                     print "argh"
1193                 return ""
1194             log = extractLogMessageFromGitCommit(next)
1195             settings = extractSettingsGitLog(log)
1196             currentChange = int(settings['change'])
1197             if self.verbose:
1198                 print "current change %s" % currentChange
1199
1200             if currentChange == change:
1201                 if self.verbose:
1202                     print "found %s" % next
1203                 return next
1204
1205             if currentChange < change:
1206                 earliestCommit = "^%s" % next
1207             else:
1208                 latestCommit = "%s" % next
1209
1210         return ""
1211
1212     def importNewBranch(self, branch, maxChange):
1213         # make fast-import flush all changes to disk and update the refs using the checkpoint
1214         # command so that we can try to find the branch parent in the git history
1215         self.gitStream.write("checkpoint\n\n");
1216         self.gitStream.flush();
1217         branchPrefix = self.depotPaths[0] + branch + "/"
1218         range = "@1,%s" % maxChange
1219         #print "prefix" + branchPrefix
1220         changes = p4ChangesForPaths([branchPrefix], range)
1221         if len(changes) <= 0:
1222             return False
1223         firstChange = changes[0]
1224         #print "first change in branch: %s" % firstChange
1225         sourceBranch = self.knownBranches[branch]
1226         sourceDepotPath = self.depotPaths[0] + sourceBranch
1227         sourceRef = self.gitRefForBranch(sourceBranch)
1228         #print "source " + sourceBranch
1229
1230         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1231         #print "branch parent: %s" % branchParentChange
1232         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1233         if len(gitParent) > 0:
1234             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1235             #print "parent git commit: %s" % gitParent
1236
1237         self.importChanges(changes)
1238         return True
1239
1240     def importChanges(self, changes):
1241         cnt = 1
1242         for change in changes:
1243             description = p4Cmd("describe %s" % change)
1244             self.updateOptionDict(description)
1245
1246             if not self.silent:
1247                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1248                 sys.stdout.flush()
1249             cnt = cnt + 1
1250
1251             try:
1252                 if self.detectBranches:
1253                     branches = self.splitFilesIntoBranches(description)
1254                     for branch in branches.keys():
1255                         ## HACK  --hwn
1256                         branchPrefix = self.depotPaths[0] + branch + "/"
1257
1258                         parent = ""
1259
1260                         filesForCommit = branches[branch]
1261
1262                         if self.verbose:
1263                             print "branch is %s" % branch
1264
1265                         self.updatedBranches.add(branch)
1266
1267                         if branch not in self.createdBranches:
1268                             self.createdBranches.add(branch)
1269                             parent = self.knownBranches[branch]
1270                             if parent == branch:
1271                                 parent = ""
1272                             else:
1273                                 fullBranch = self.projectName + branch
1274                                 if fullBranch not in self.p4BranchesInGit:
1275                                     if not self.silent:
1276                                         print("\n    Importing new branch %s" % fullBranch);
1277                                     if self.importNewBranch(branch, change - 1):
1278                                         parent = ""
1279                                         self.p4BranchesInGit.append(fullBranch)
1280                                     if not self.silent:
1281                                         print("\n    Resuming with change %s" % change);
1282
1283                                 if self.verbose:
1284                                     print "parent determined through known branches: %s" % parent
1285
1286                         branch = self.gitRefForBranch(branch)
1287                         parent = self.gitRefForBranch(parent)
1288
1289                         if self.verbose:
1290                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1291
1292                         if len(parent) == 0 and branch in self.initialParents:
1293                             parent = self.initialParents[branch]
1294                             del self.initialParents[branch]
1295
1296                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1297                 else:
1298                     files = self.extractFilesFromCommit(description)
1299                     self.commit(description, files, self.branch, self.depotPaths,
1300                                 self.initialParent)
1301                     self.initialParent = ""
1302             except IOError:
1303                 print self.gitError.read()
1304                 sys.exit(1)
1305
1306     def importHeadRevision(self, revision):
1307         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1308
1309         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1310         details["desc"] = ("Initial import of %s from the state at revision %s"
1311                            % (' '.join(self.depotPaths), revision))
1312         details["change"] = revision
1313         newestRevision = 0
1314
1315         fileCnt = 0
1316         for info in p4CmdList("files "
1317                               +  ' '.join(["%s...%s"
1318                                            % (p, revision)
1319                                            for p in self.depotPaths])):
1320
1321             if info['code'] == 'error':
1322                 sys.stderr.write("p4 returned an error: %s\n"
1323                                  % info['data'])
1324                 sys.exit(1)
1325
1326
1327             change = int(info["change"])
1328             if change > newestRevision:
1329                 newestRevision = change
1330
1331             if info["action"] == "delete":
1332                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1333                 #fileCnt = fileCnt + 1
1334                 continue
1335
1336             for prop in ["depotFile", "rev", "action", "type" ]:
1337                 details["%s%s" % (prop, fileCnt)] = info[prop]
1338
1339             fileCnt = fileCnt + 1
1340
1341         details["change"] = newestRevision
1342         self.updateOptionDict(details)
1343         try:
1344             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1345         except IOError:
1346             print "IO error with git fast-import. Is your git version recent enough?"
1347             print self.gitError.read()
1348
1349
1350     def getClientSpec(self):
1351         specList = p4CmdList( "client -o" )
1352         temp = {}
1353         for entry in specList:
1354             for k,v in entry.iteritems():
1355                 if k.startswith("View"):
1356                     if v.startswith('"'):
1357                         start = 1
1358                     else:
1359                         start = 0
1360                     index = v.find("...")
1361                     v = v[start:index]
1362                     if v.startswith("-"):
1363                         v = v[1:]
1364                         temp[v] = -len(v)
1365                     else:
1366                         temp[v] = len(v)
1367         self.clientSpecDirs = temp.items()
1368         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1369
1370     def run(self, args):
1371         self.depotPaths = []
1372         self.changeRange = ""
1373         self.initialParent = ""
1374         self.previousDepotPaths = []
1375
1376         # map from branch depot path to parent branch
1377         self.knownBranches = {}
1378         self.initialParents = {}
1379         self.hasOrigin = originP4BranchesExist()
1380         if not self.syncWithOrigin:
1381             self.hasOrigin = False
1382
1383         if self.importIntoRemotes:
1384             self.refPrefix = "refs/remotes/p4/"
1385         else:
1386             self.refPrefix = "refs/heads/p4/"
1387
1388         if self.syncWithOrigin and self.hasOrigin:
1389             if not self.silent:
1390                 print "Syncing with origin first by calling git fetch origin"
1391             system("git fetch origin")
1392
1393         if len(self.branch) == 0:
1394             self.branch = self.refPrefix + "master"
1395             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1396                 system("git update-ref %s refs/heads/p4" % self.branch)
1397                 system("git branch -D p4");
1398             # create it /after/ importing, when master exists
1399             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1400                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1401
1402         if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1403             self.getClientSpec()
1404
1405         # TODO: should always look at previous commits,
1406         # merge with previous imports, if possible.
1407         if args == []:
1408             if self.hasOrigin:
1409                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1410             self.listExistingP4GitBranches()
1411
1412             if len(self.p4BranchesInGit) > 1:
1413                 if not self.silent:
1414                     print "Importing from/into multiple branches"
1415                 self.detectBranches = True
1416
1417             if self.verbose:
1418                 print "branches: %s" % self.p4BranchesInGit
1419
1420             p4Change = 0
1421             for branch in self.p4BranchesInGit:
1422                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1423
1424                 settings = extractSettingsGitLog(logMsg)
1425
1426                 self.readOptions(settings)
1427                 if (settings.has_key('depot-paths')
1428                     and settings.has_key ('change')):
1429                     change = int(settings['change']) + 1
1430                     p4Change = max(p4Change, change)
1431
1432                     depotPaths = sorted(settings['depot-paths'])
1433                     if self.previousDepotPaths == []:
1434                         self.previousDepotPaths = depotPaths
1435                     else:
1436                         paths = []
1437                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1438                             for i in range(0, min(len(cur), len(prev))):
1439                                 if cur[i] <> prev[i]:
1440                                     i = i - 1
1441                                     break
1442
1443                             paths.append (cur[:i + 1])
1444
1445                         self.previousDepotPaths = paths
1446
1447             if p4Change > 0:
1448                 self.depotPaths = sorted(self.previousDepotPaths)
1449                 self.changeRange = "@%s,#head" % p4Change
1450                 if not self.detectBranches:
1451                     self.initialParent = parseRevision(self.branch)
1452                 if not self.silent and not self.detectBranches:
1453                     print "Performing incremental import into %s git branch" % self.branch
1454
1455         if not self.branch.startswith("refs/"):
1456             self.branch = "refs/heads/" + self.branch
1457
1458         if len(args) == 0 and self.depotPaths:
1459             if not self.silent:
1460                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1461         else:
1462             if self.depotPaths and self.depotPaths != args:
1463                 print ("previous import used depot path %s and now %s was specified. "
1464                        "This doesn't work!" % (' '.join (self.depotPaths),
1465                                                ' '.join (args)))
1466                 sys.exit(1)
1467
1468             self.depotPaths = sorted(args)
1469
1470         revision = ""
1471         self.users = {}
1472
1473         newPaths = []
1474         for p in self.depotPaths:
1475             if p.find("@") != -1:
1476                 atIdx = p.index("@")
1477                 self.changeRange = p[atIdx:]
1478                 if self.changeRange == "@all":
1479                     self.changeRange = ""
1480                 elif ',' not in self.changeRange:
1481                     revision = self.changeRange
1482                     self.changeRange = ""
1483                 p = p[:atIdx]
1484             elif p.find("#") != -1:
1485                 hashIdx = p.index("#")
1486                 revision = p[hashIdx:]
1487                 p = p[:hashIdx]
1488             elif self.previousDepotPaths == []:
1489                 revision = "#head"
1490
1491             p = re.sub ("\.\.\.$", "", p)
1492             if not p.endswith("/"):
1493                 p += "/"
1494
1495             newPaths.append(p)
1496
1497         self.depotPaths = newPaths
1498
1499
1500         self.loadUserMapFromCache()
1501         self.labels = {}
1502         if self.detectLabels:
1503             self.getLabels();
1504
1505         if self.detectBranches:
1506             ## FIXME - what's a P4 projectName ?
1507             self.projectName = self.guessProjectName()
1508
1509             if self.hasOrigin:
1510                 self.getBranchMappingFromGitBranches()
1511             else:
1512                 self.getBranchMapping()
1513             if self.verbose:
1514                 print "p4-git branches: %s" % self.p4BranchesInGit
1515                 print "initial parents: %s" % self.initialParents
1516             for b in self.p4BranchesInGit:
1517                 if b != "master":
1518
1519                     ## FIXME
1520                     b = b[len(self.projectName):]
1521                 self.createdBranches.add(b)
1522
1523         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1524
1525         importProcess = subprocess.Popen(["git", "fast-import"],
1526                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1527                                          stderr=subprocess.PIPE);
1528         self.gitOutput = importProcess.stdout
1529         self.gitStream = importProcess.stdin
1530         self.gitError = importProcess.stderr
1531
1532         if revision:
1533             self.importHeadRevision(revision)
1534         else:
1535             changes = []
1536
1537             if len(self.changesFile) > 0:
1538                 output = open(self.changesFile).readlines()
1539                 changeSet = Set()
1540                 for line in output:
1541                     changeSet.add(int(line))
1542
1543                 for change in changeSet:
1544                     changes.append(change)
1545
1546                 changes.sort()
1547             else:
1548                 if self.verbose:
1549                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1550                                                               self.changeRange)
1551                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1552
1553                 if len(self.maxChanges) > 0:
1554                     changes = changes[:min(int(self.maxChanges), len(changes))]
1555
1556             if len(changes) == 0:
1557                 if not self.silent:
1558                     print "No changes to import!"
1559                 return True
1560
1561             if not self.silent and not self.detectBranches:
1562                 print "Import destination: %s" % self.branch
1563
1564             self.updatedBranches = set()
1565
1566             self.importChanges(changes)
1567
1568             if not self.silent:
1569                 print ""
1570                 if len(self.updatedBranches) > 0:
1571                     sys.stdout.write("Updated branches: ")
1572                     for b in self.updatedBranches:
1573                         sys.stdout.write("%s " % b)
1574                     sys.stdout.write("\n")
1575
1576         self.gitStream.close()
1577         if importProcess.wait() != 0:
1578             die("fast-import failed: %s" % self.gitError.read())
1579         self.gitOutput.close()
1580         self.gitError.close()
1581
1582         return True
1583
1584 class P4Rebase(Command):
1585     def __init__(self):
1586         Command.__init__(self)
1587         self.options = [ ]
1588         self.description = ("Fetches the latest revision from perforce and "
1589                             + "rebases the current work (branch) against it")
1590         self.verbose = False
1591
1592     def run(self, args):
1593         sync = P4Sync()
1594         sync.run([])
1595
1596         return self.rebase()
1597
1598     def rebase(self):
1599         if os.system("git update-index --refresh") != 0:
1600             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1601         if len(read_pipe("git diff-index HEAD --")) > 0:
1602             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1603
1604         [upstream, settings] = findUpstreamBranchPoint()
1605         if len(upstream) == 0:
1606             die("Cannot find upstream branchpoint for rebase")
1607
1608         # the branchpoint may be p4/foo~3, so strip off the parent
1609         upstream = re.sub("~[0-9]+$", "", upstream)
1610
1611         print "Rebasing the current branch onto %s" % upstream
1612         oldHead = read_pipe("git rev-parse HEAD").strip()
1613         system("git rebase %s" % upstream)
1614         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1615         return True
1616
1617 class P4Clone(P4Sync):
1618     def __init__(self):
1619         P4Sync.__init__(self)
1620         self.description = "Creates a new git repository and imports from Perforce into it"
1621         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1622         self.options += [
1623             optparse.make_option("--destination", dest="cloneDestination",
1624                                  action='store', default=None,
1625                                  help="where to leave result of the clone"),
1626             optparse.make_option("-/", dest="cloneExclude",
1627                                  action="append", type="string",
1628                                  help="exclude depot path")
1629         ]
1630         self.cloneDestination = None
1631         self.needsGit = False
1632
1633     # This is required for the "append" cloneExclude action
1634     def ensure_value(self, attr, value):
1635         if not hasattr(self, attr) or getattr(self, attr) is None:
1636             setattr(self, attr, value)
1637         return getattr(self, attr)
1638
1639     def defaultDestination(self, args):
1640         ## TODO: use common prefix of args?
1641         depotPath = args[0]
1642         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1643         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1644         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1645         depotDir = re.sub(r"/$", "", depotDir)
1646         return os.path.split(depotDir)[1]
1647
1648     def run(self, args):
1649         if len(args) < 1:
1650             return False
1651
1652         if self.keepRepoPath and not self.cloneDestination:
1653             sys.stderr.write("Must specify destination for --keep-path\n")
1654             sys.exit(1)
1655
1656         depotPaths = args
1657
1658         if not self.cloneDestination and len(depotPaths) > 1:
1659             self.cloneDestination = depotPaths[-1]
1660             depotPaths = depotPaths[:-1]
1661
1662         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1663         for p in depotPaths:
1664             if not p.startswith("//"):
1665                 return False
1666
1667         if not self.cloneDestination:
1668             self.cloneDestination = self.defaultDestination(args)
1669
1670         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1671         if not os.path.exists(self.cloneDestination):
1672             os.makedirs(self.cloneDestination)
1673         os.chdir(self.cloneDestination)
1674         system("git init")
1675         self.gitdir = os.getcwd() + "/.git"
1676         if not P4Sync.run(self, depotPaths):
1677             return False
1678         if self.branch != "master":
1679             if gitBranchExists("refs/remotes/p4/master"):
1680                 system("git branch master refs/remotes/p4/master")
1681                 system("git checkout -f")
1682             else:
1683                 print "Could not detect main branch. No checkout/master branch created."
1684
1685         return True
1686
1687 class P4Branches(Command):
1688     def __init__(self):
1689         Command.__init__(self)
1690         self.options = [ ]
1691         self.description = ("Shows the git branches that hold imports and their "
1692                             + "corresponding perforce depot paths")
1693         self.verbose = False
1694
1695     def run(self, args):
1696         if originP4BranchesExist():
1697             createOrUpdateBranchesFromOrigin()
1698
1699         cmdline = "git rev-parse --symbolic "
1700         cmdline += " --remotes"
1701
1702         for line in read_pipe_lines(cmdline):
1703             line = line.strip()
1704
1705             if not line.startswith('p4/') or line == "p4/HEAD":
1706                 continue
1707             branch = line
1708
1709             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1710             settings = extractSettingsGitLog(log)
1711
1712             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1713         return True
1714
1715 class HelpFormatter(optparse.IndentedHelpFormatter):
1716     def __init__(self):
1717         optparse.IndentedHelpFormatter.__init__(self)
1718
1719     def format_description(self, description):
1720         if description:
1721             return description + "\n"
1722         else:
1723             return ""
1724
1725 def printUsage(commands):
1726     print "usage: %s <command> [options]" % sys.argv[0]
1727     print ""
1728     print "valid commands: %s" % ", ".join(commands)
1729     print ""
1730     print "Try %s <command> --help for command specific help." % sys.argv[0]
1731     print ""
1732
1733 commands = {
1734     "debug" : P4Debug,
1735     "submit" : P4Submit,
1736     "commit" : P4Submit,
1737     "sync" : P4Sync,
1738     "rebase" : P4Rebase,
1739     "clone" : P4Clone,
1740     "rollback" : P4RollBack,
1741     "branches" : P4Branches
1742 }
1743
1744
1745 def main():
1746     if len(sys.argv[1:]) == 0:
1747         printUsage(commands.keys())
1748         sys.exit(2)
1749
1750     cmd = ""
1751     cmdName = sys.argv[1]
1752     try:
1753         klass = commands[cmdName]
1754         cmd = klass()
1755     except KeyError:
1756         print "unknown command %s" % cmdName
1757         print ""
1758         printUsage(commands.keys())
1759         sys.exit(2)
1760
1761     options = cmd.options
1762     cmd.gitdir = os.environ.get("GIT_DIR", None)
1763
1764     args = sys.argv[2:]
1765
1766     if len(options) > 0:
1767         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1768
1769         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1770                                        options,
1771                                        description = cmd.description,
1772                                        formatter = HelpFormatter())
1773
1774         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1775     global verbose
1776     verbose = cmd.verbose
1777     if cmd.needsGit:
1778         if cmd.gitdir == None:
1779             cmd.gitdir = os.path.abspath(".git")
1780             if not isValidGitDir(cmd.gitdir):
1781                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1782                 if os.path.exists(cmd.gitdir):
1783                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1784                     if len(cdup) > 0:
1785                         os.chdir(cdup);
1786
1787         if not isValidGitDir(cmd.gitdir):
1788             if isValidGitDir(cmd.gitdir + "/.git"):
1789                 cmd.gitdir += "/.git"
1790             else:
1791                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1792
1793         os.environ["GIT_DIR"] = cmd.gitdir
1794
1795     if not cmd.run(args):
1796         parser.print_help()
1797
1798
1799 if __name__ == '__main__':
1800     main()