contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def chdir(dir):
  20     if os.name == 'nt':
  21         os.environ['PWD']=dir
  22     os.chdir(dir)
  23
  24 def die(msg):
  25     if verbose:
  26         raise Exception(msg)
  27     else:
  28         sys.stderr.write(msg + "\n")
  29         sys.exit(1)
  30
  31 def write_pipe(c, str):
  32     if verbose:
  33         sys.stderr.write('Writing pipe: %s\n' % c)
  34
  35     pipe = os.popen(c, 'w')
  36     val = pipe.write(str)
  37     if pipe.close():
  38         die('Command failed: %s' % c)
  39
  40     return val
  41
  42 def read_pipe(c, ignore_error=False):
  43     if verbose:
  44         sys.stderr.write('Reading pipe: %s\n' % c)
  45
  46     pipe = os.popen(c, 'rb')
  47     val = pipe.read()
  48     if pipe.close() and not ignore_error:
  49         die('Command failed: %s' % c)
  50
  51     return val
  52
  53
  54 def read_pipe_lines(c):
  55     if verbose:
  56         sys.stderr.write('Reading pipe: %s\n' % c)
  57     ## todo: check return status
  58     pipe = os.popen(c, 'rb')
  59     val = pipe.readlines()
  60     if pipe.close():
  61         die('Command failed: %s' % c)
  62
  63     return val
  64
  65 def system(cmd):
  66     if verbose:
  67         sys.stderr.write("executing %s\n" % cmd)
  68     if os.system(cmd) != 0:
  69         die("command failed: %s" % cmd)
  70
  71 def isP4Exec(kind):
  72     """Determine if a Perforce 'kind' should have execute permission
  73
  74     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  75     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  76     a plus sign, it is also executable"""
  77     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  78
  79 def setP4ExecBit(file, mode):
  80     # Reopens an already open file and changes the execute bit to match
  81     # the execute bit setting in the passed in mode.
  82
  83     p4Type = "+x"
  84
  85     if not isModeExec(mode):
  86         p4Type = getP4OpenedType(file)
  87         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  88         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  89         if p4Type[-1] == "+":
  90             p4Type = p4Type[0:-1]
  91
  92     system("p4 reopen -t %s %s" % (p4Type, file))
  93
  94 def getP4OpenedType(file):
  95     # Returns the perforce file type for the given file.
  96
  97     result = read_pipe("p4 opened %s" % file)
  98     match = re.match(".*\((.+)\)\r?$", result)
  99     if match:
 100         return match.group(1)
 101     else:
 102         die("Could not determine file type for %s (result: '%s')" % (file, result))
 103
 104 def diffTreePattern():
 105     # This is a simple generator for the diff tree regex pattern. This could be
 106     # a class variable if this and parseDiffTreeEntry were a part of a class.
 107     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 108     while True:
 109         yield pattern
 110
 111 def parseDiffTreeEntry(entry):
 112     """Parses a single diff tree entry into its component elements.
 113
 114     See git-diff-tree(1) manpage for details about the format of the diff
 115     output. This method returns a dictionary with the following elements:
 116
 117     src_mode - The mode of the source file
 118     dst_mode - The mode of the destination file
 119     src_sha1 - The sha1 for the source file
 120     dst_sha1 - The sha1 fr the destination file
 121     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 122     status_score - The score for the status (applicable for 'C' and 'R'
 123                    statuses). This is None if there is no score.
 124     src - The path for the source file.
 125     dst - The path for the destination file. This is only present for
 126           copy or renames. If it is not present, this is None.
 127
 128     If the pattern is not matched, None is returned."""
 129
 130     match = diffTreePattern().next().match(entry)
 131     if match:
 132         return {
 133             'src_mode': match.group(1),
 134             'dst_mode': match.group(2),
 135             'src_sha1': match.group(3),
 136             'dst_sha1': match.group(4),
 137             'status': match.group(5),
 138             'status_score': match.group(6),
 139             'src': match.group(7),
 140             'dst': match.group(10)
 141         }
 142     return None
 143
 144 def isModeExec(mode):
 145     # Returns True if the given git mode represents an executable file,
 146     # otherwise False.
 147     return mode[-3:] == "755"
 148
 149 def isModeExecChanged(src_mode, dst_mode):
 150     return isModeExec(src_mode) != isModeExec(dst_mode)
 151
 152 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 153     cmd = "p4 -G %s" % cmd
 154     if verbose:
 155         sys.stderr.write("Opening pipe: %s\n" % cmd)
 156
 157     # Use a temporary file to avoid deadlocks without
 158     # subprocess.communicate(), which would put another copy
 159     # of stdout into memory.
 160     stdin_file = None
 161     if stdin is not None:
 162         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 163         stdin_file.write(stdin)
 164         stdin_file.flush()
 165         stdin_file.seek(0)
 166
 167     p4 = subprocess.Popen(cmd, shell=True,
 168                           stdin=stdin_file,
 169                           stdout=subprocess.PIPE)
 170
 171     result = []
 172     try:
 173         while True:
 174             entry = marshal.load(p4.stdout)
 175             result.append(entry)
 176     except EOFError:
 177         pass
 178     exitCode = p4.wait()
 179     if exitCode != 0:
 180         entry = {}
 181         entry["p4ExitCode"] = exitCode
 182         result.append(entry)
 183
 184     return result
 185
 186 def p4Cmd(cmd):
 187     list = p4CmdList(cmd)
 188     result = {}
 189     for entry in list:
 190         result.update(entry)
 191     return result;
 192
 193 def p4Where(depotPath):
 194     if not depotPath.endswith("/"):
 195         depotPath += "/"
 196     output = p4Cmd("where %s..." % depotPath)
 197     if output["code"] == "error":
 198         return ""
 199     clientPath = ""
 200     if "path" in output:
 201         clientPath = output.get("path")
 202     elif "data" in output:
 203         data = output.get("data")
 204         lastSpace = data.rfind(" ")
 205         clientPath = data[lastSpace + 1:]
 206
 207     if clientPath.endswith("..."):
 208         clientPath = clientPath[:-3]
 209     return clientPath
 210
 211 def currentGitBranch():
 212     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 213
 214 def isValidGitDir(path):
 215     if (os.path.exists(path + "/HEAD")
 216         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 217         return True;
 218     return False
 219
 220 def parseRevision(ref):
 221     return read_pipe("git rev-parse %s" % ref).strip()
 222
 223 def extractLogMessageFromGitCommit(commit):
 224     logMessage = ""
 225
 226     ## fixme: title is first line of commit, not 1st paragraph.
 227     foundTitle = False
 228     for log in read_pipe_lines("git cat-file commit %s" % commit):
 229        if not foundTitle:
 230            if len(log) == 1:
 231                foundTitle = True
 232            continue
 233
 234        logMessage += log
 235     return logMessage
 236
 237 def extractSettingsGitLog(log):
 238     values = {}
 239     for line in log.split("\n"):
 240         line = line.strip()
 241         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 242         if not m:
 243             continue
 244
 245         assignments = m.group(1).split (':')
 246         for a in assignments:
 247             vals = a.split ('=')
 248             key = vals[0].strip()
 249             val = ('='.join (vals[1:])).strip()
 250             if val.endswith ('\"') and val.startswith('"'):
 251                 val = val[1:-1]
 252
 253             values[key] = val
 254
 255     paths = values.get("depot-paths")
 256     if not paths:
 257         paths = values.get("depot-path")
 258     if paths:
 259         values['depot-paths'] = paths.split(',')
 260     return values
 261
 262 def gitBranchExists(branch):
 263     proc = subprocess.Popen(["git", "rev-parse", branch],
 264                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 265     return proc.wait() == 0;
 266
 267 def gitConfig(key):
 268     return read_pipe("git config %s" % key, ignore_error=True).strip()
 269
 270 def p4BranchesInGit(branchesAreInRemotes = True):
 271     branches = {}
 272
 273     cmdline = "git rev-parse --symbolic "
 274     if branchesAreInRemotes:
 275         cmdline += " --remotes"
 276     else:
 277         cmdline += " --branches"
 278
 279     for line in read_pipe_lines(cmdline):
 280         line = line.strip()
 281
 282         ## only import to p4/
 283         if not line.startswith('p4/') or line == "p4/HEAD":
 284             continue
 285         branch = line
 286
 287         # strip off p4
 288         branch = re.sub ("^p4/", "", line)
 289
 290         branches[branch] = parseRevision(line)
 291     return branches
 292
 293 def findUpstreamBranchPoint(head = "HEAD"):
 294     branches = p4BranchesInGit()
 295     # map from depot-path to branch name
 296     branchByDepotPath = {}
 297     for branch in branches.keys():
 298         tip = branches[branch]
 299         log = extractLogMessageFromGitCommit(tip)
 300         settings = extractSettingsGitLog(log)
 301         if settings.has_key("depot-paths"):
 302             paths = ",".join(settings["depot-paths"])
 303             branchByDepotPath[paths] = "remotes/p4/" + branch
 304
 305     settings = None
 306     parent = 0
 307     while parent < 65535:
 308         commit = head + "~%s" % parent
 309         log = extractLogMessageFromGitCommit(commit)
 310         settings = extractSettingsGitLog(log)
 311         if settings.has_key("depot-paths"):
 312             paths = ",".join(settings["depot-paths"])
 313             if branchByDepotPath.has_key(paths):
 314                 return [branchByDepotPath[paths], settings]
 315
 316         parent = parent + 1
 317
 318     return ["", settings]
 319
 320 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 321     if not silent:
 322         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 323                % localRefPrefix)
 324
 325     originPrefix = "origin/p4/"
 326
 327     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 328         line = line.strip()
 329         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 330             continue
 331
 332         headName = line[len(originPrefix):]
 333         remoteHead = localRefPrefix + headName
 334         originHead = line
 335
 336         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 337         if (not original.has_key('depot-paths')
 338             or not original.has_key('change')):
 339             continue
 340
 341         update = False
 342         if not gitBranchExists(remoteHead):
 343             if verbose:
 344                 print "creating %s" % remoteHead
 345             update = True
 346         else:
 347             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 348             if settings.has_key('change') > 0:
 349                 if settings['depot-paths'] == original['depot-paths']:
 350                     originP4Change = int(original['change'])
 351                     p4Change = int(settings['change'])
 352                     if originP4Change > p4Change:
 353                         print ("%s (%s) is newer than %s (%s). "
 354                                "Updating p4 branch from origin."
 355                                % (originHead, originP4Change,
 356                                   remoteHead, p4Change))
 357                         update = True
 358                 else:
 359                     print ("Ignoring: %s was imported from %s while "
 360                            "%s was imported from %s"
 361                            % (originHead, ','.join(original['depot-paths']),
 362                               remoteHead, ','.join(settings['depot-paths'])))
 363
 364         if update:
 365             system("git update-ref %s %s" % (remoteHead, originHead))
 366
 367 def originP4BranchesExist():
 368         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 369
 370 def p4ChangesForPaths(depotPaths, changeRange):
 371     assert depotPaths
 372     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 373                                                         for p in depotPaths]))
 374
 375     changes = []
 376     for line in output:
 377         changeNum = line.split(" ")[1]
 378         changes.append(int(changeNum))
 379
 380     changes.sort()
 381     return changes
 382
 383 class Command:
 384     def __init__(self):
 385         self.usage = "usage: %prog [options]"
 386         self.needsGit = True
 387
 388 class P4Debug(Command):
 389     def __init__(self):
 390         Command.__init__(self)
 391         self.options = [
 392             optparse.make_option("--verbose", dest="verbose", action="store_true",
 393                                  default=False),
 394             ]
 395         self.description = "A tool to debug the output of p4 -G."
 396         self.needsGit = False
 397         self.verbose = False
 398
 399     def run(self, args):
 400         j = 0
 401         for output in p4CmdList(" ".join(args)):
 402             print 'Element: %d' % j
 403             j += 1
 404             print output
 405         return True
 406
 407 class P4RollBack(Command):
 408     def __init__(self):
 409         Command.__init__(self)
 410         self.options = [
 411             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 412             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 413         ]
 414         self.description = "A tool to debug the multi-branch import. Don't use :)"
 415         self.verbose = False
 416         self.rollbackLocalBranches = False
 417
 418     def run(self, args):
 419         if len(args) != 1:
 420             return False
 421         maxChange = int(args[0])
 422
 423         if "p4ExitCode" in p4Cmd("changes -m 1"):
 424             die("Problems executing p4");
 425
 426         if self.rollbackLocalBranches:
 427             refPrefix = "refs/heads/"
 428             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 429         else:
 430             refPrefix = "refs/remotes/"
 431             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 432
 433         for line in lines:
 434             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 435                 line = line.strip()
 436                 ref = refPrefix + line
 437                 log = extractLogMessageFromGitCommit(ref)
 438                 settings = extractSettingsGitLog(log)
 439
 440                 depotPaths = settings['depot-paths']
 441                 change = settings['change']
 442
 443                 changed = False
 444
 445                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 446                                                            for p in depotPaths]))) == 0:
 447                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 448                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 449                     continue
 450
 451                 while change and int(change) > maxChange:
 452                     changed = True
 453                     if self.verbose:
 454                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 455                     system("git update-ref %s \"%s^\"" % (ref, ref))
 456                     log = extractLogMessageFromGitCommit(ref)
 457                     settings =  extractSettingsGitLog(log)
 458
 459
 460                     depotPaths = settings['depot-paths']
 461                     change = settings['change']
 462
 463                 if changed:
 464                     print "%s rewound to %s" % (ref, change)
 465
 466         return True
 467
 468 class P4Submit(Command):
 469     def __init__(self):
 470         Command.__init__(self)
 471         self.options = [
 472                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 473                 optparse.make_option("--origin", dest="origin"),
 474                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 475         ]
 476         self.description = "Submit changes from git to the perforce depot."
 477         self.usage += " [name of git branch to submit into perforce depot]"
 478         self.interactive = True
 479         self.origin = ""
 480         self.detectRename = False
 481         self.verbose = False
 482         self.isWindows = (platform.system() == "Windows")
 483
 484     def check(self):
 485         if len(p4CmdList("opened ...")) > 0:
 486             die("You have files opened with perforce! Close them before starting the sync.")
 487
 488     # replaces everything between 'Description:' and the next P4 submit template field with the
 489     # commit message
 490     def prepareLogMessage(self, template, message):
 491         result = ""
 492
 493         inDescriptionSection = False
 494
 495         for line in template.split("\n"):
 496             if line.startswith("#"):
 497                 result += line + "\n"
 498                 continue
 499
 500             if inDescriptionSection:
 501                 if line.startswith("Files:"):
 502                     inDescriptionSection = False
 503                 else:
 504                     continue
 505             else:
 506                 if line.startswith("Description:"):
 507                     inDescriptionSection = True
 508                     line += "\n"
 509                     for messageLine in message.split("\n"):
 510                         line += "\t" + messageLine + "\n"
 511
 512             result += line + "\n"
 513
 514         return result
 515
 516     def prepareSubmitTemplate(self):
 517         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 518         template = ""
 519         inFilesSection = False
 520         for line in read_pipe_lines("p4 change -o"):
 521             if line.endswith("\r\n"):
 522                 line = line[:-2] + "\n"
 523             if inFilesSection:
 524                 if line.startswith("\t"):
 525                     # path starts and ends with a tab
 526                     path = line[1:]
 527                     lastTab = path.rfind("\t")
 528                     if lastTab != -1:
 529                         path = path[:lastTab]
 530                         if not path.startswith(self.depotPath):
 531                             continue
 532                 else:
 533                     inFilesSection = False
 534             else:
 535                 if line.startswith("Files:"):
 536                     inFilesSection = True
 537
 538             template += line
 539
 540         return template
 541
 542     def applyCommit(self, id):
 543         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 544         diffOpts = ("", "-M")[self.detectRename]
 545         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 546         filesToAdd = set()
 547         filesToDelete = set()
 548         editedFiles = set()
 549         filesToChangeExecBit = {}
 550         for line in diff:
 551             diff = parseDiffTreeEntry(line)
 552             modifier = diff['status']
 553             path = diff['src']
 554             if modifier == "M":
 555                 system("p4 edit \"%s\"" % path)
 556                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 557                     filesToChangeExecBit[path] = diff['dst_mode']
 558                 editedFiles.add(path)
 559             elif modifier == "A":
 560                 filesToAdd.add(path)
 561                 filesToChangeExecBit[path] = diff['dst_mode']
 562                 if path in filesToDelete:
 563                     filesToDelete.remove(path)
 564             elif modifier == "D":
 565                 filesToDelete.add(path)
 566                 if path in filesToAdd:
 567                     filesToAdd.remove(path)
 568             elif modifier == "R":
 569                 src, dest = diff['src'], diff['dst']
 570                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 571                 system("p4 edit \"%s\"" % (dest))
 572                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 573                     filesToChangeExecBit[dest] = diff['dst_mode']
 574                 os.unlink(dest)
 575                 editedFiles.add(dest)
 576                 filesToDelete.add(src)
 577             else:
 578                 die("unknown modifier %s for %s" % (modifier, path))
 579
 580         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 581         patchcmd = diffcmd + " | git apply "
 582         tryPatchCmd = patchcmd + "--check -"
 583         applyPatchCmd = patchcmd + "--check --apply -"
 584
 585         if os.system(tryPatchCmd) != 0:
 586             print "Unfortunately applying the change failed!"
 587             print "What do you want to do?"
 588             response = "x"
 589             while response != "s" and response != "a" and response != "w":
 590                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 591                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 592             if response == "s":
 593                 print "Skipping! Good luck with the next patches..."
 594                 for f in editedFiles:
 595                     system("p4 revert \"%s\"" % f);
 596                 for f in filesToAdd:
 597                     system("rm %s" %f)
 598                 return
 599             elif response == "a":
 600                 os.system(applyPatchCmd)
 601                 if len(filesToAdd) > 0:
 602                     print "You may also want to call p4 add on the following files:"
 603                     print " ".join(filesToAdd)
 604                 if len(filesToDelete):
 605                     print "The following files should be scheduled for deletion with p4 delete:"
 606                     print " ".join(filesToDelete)
 607                 die("Please resolve and submit the conflict manually and "
 608                     + "continue afterwards with git-p4 submit --continue")
 609             elif response == "w":
 610                 system(diffcmd + " > patch.txt")
 611                 print "Patch saved to patch.txt in %s !" % self.clientPath
 612                 die("Please resolve and submit the conflict manually and "
 613                     "continue afterwards with git-p4 submit --continue")
 614
 615         system(applyPatchCmd)
 616
 617         for f in filesToAdd:
 618             system("p4 add \"%s\"" % f)
 619         for f in filesToDelete:
 620             system("p4 revert \"%s\"" % f)
 621             system("p4 delete \"%s\"" % f)
 622
 623         # Set/clear executable bits
 624         for f in filesToChangeExecBit.keys():
 625             mode = filesToChangeExecBit[f]
 626             setP4ExecBit(f, mode)
 627
 628         logMessage = extractLogMessageFromGitCommit(id)
 629         logMessage = logMessage.strip()
 630
 631         template = self.prepareSubmitTemplate()
 632
 633         if self.interactive:
 634             submitTemplate = self.prepareLogMessage(template, logMessage)
 635             if os.environ.has_key("P4DIFF"):
 636                 del(os.environ["P4DIFF"])
 637             diff = read_pipe("p4 diff -du ...")
 638
 639             newdiff = ""
 640             for newFile in filesToAdd:
 641                 newdiff += "==== new file ====\n"
 642                 newdiff += "--- /dev/null\n"
 643                 newdiff += "+++ %s\n" % newFile
 644                 f = open(newFile, "r")
 645                 for line in f.readlines():
 646                     newdiff += "+" + line
 647                 f.close()
 648
 649             separatorLine = "######## everything below this line is just the diff #######\n"
 650
 651             [handle, fileName] = tempfile.mkstemp()
 652             tmpFile = os.fdopen(handle, "w+")
 653             if self.isWindows:
 654                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 655                 separatorLine = separatorLine.replace("\n", "\r\n")
 656                 newdiff = newdiff.replace("\n", "\r\n")
 657             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 658             tmpFile.close()
 659             defaultEditor = "vi"
 660             if platform.system() == "Windows":
 661                 defaultEditor = "notepad"
 662             if os.environ.has_key("P4EDITOR"):
 663                 editor = os.environ.get("P4EDITOR")
 664             else:
 665                 editor = os.environ.get("EDITOR", defaultEditor);
 666             system(editor + " " + fileName)
 667             tmpFile = open(fileName, "rb")
 668             message = tmpFile.read()
 669             tmpFile.close()
 670             os.remove(fileName)
 671             submitTemplate = message[:message.index(separatorLine)]
 672             if self.isWindows:
 673                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 674
 675             write_pipe("p4 submit -i", submitTemplate)
 676         else:
 677             fileName = "submit.txt"
 678             file = open(fileName, "w+")
 679             file.write(self.prepareLogMessage(template, logMessage))
 680             file.close()
 681             print ("Perforce submit template written as %s. "
 682                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 683                    % (fileName, fileName))
 684
 685     def run(self, args):
 686         if len(args) == 0:
 687             self.master = currentGitBranch()
 688             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 689                 die("Detecting current git branch failed!")
 690         elif len(args) == 1:
 691             self.master = args[0]
 692         else:
 693             return False
 694
 695         allowSubmit = gitConfig("git-p4.allowSubmit")
 696         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 697             die("%s is not in git-p4.allowSubmit" % self.master)
 698
 699         [upstream, settings] = findUpstreamBranchPoint()
 700         self.depotPath = settings['depot-paths'][0]
 701         if len(self.origin) == 0:
 702             self.origin = upstream
 703
 704         if self.verbose:
 705             print "Origin branch is " + self.origin
 706
 707         if len(self.depotPath) == 0:
 708             print "Internal error: cannot locate perforce depot path from existing branches"
 709             sys.exit(128)
 710
 711         self.clientPath = p4Where(self.depotPath)
 712
 713         if len(self.clientPath) == 0:
 714             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 715             sys.exit(128)
 716
 717         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 718         self.oldWorkingDirectory = os.getcwd()
 719
 720         chdir(self.clientPath)
 721         print "Syncronizing p4 checkout..."
 722         system("p4 sync ...")
 723
 724         self.check()
 725
 726         commits = []
 727         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 728             commits.append(line.strip())
 729         commits.reverse()
 730
 731         while len(commits) > 0:
 732             commit = commits[0]
 733             commits = commits[1:]
 734             self.applyCommit(commit)
 735             if not self.interactive:
 736                 break
 737
 738         if len(commits) == 0:
 739             print "All changes applied!"
 740             chdir(self.oldWorkingDirectory)
 741
 742             sync = P4Sync()
 743             sync.run([])
 744
 745             rebase = P4Rebase()
 746             rebase.rebase()
 747
 748         return True
 749
 750 class P4Sync(Command):
 751     def __init__(self):
 752         Command.__init__(self)
 753         self.options = [
 754                 optparse.make_option("--branch", dest="branch"),
 755                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 756                 optparse.make_option("--changesfile", dest="changesFile"),
 757                 optparse.make_option("--silent", dest="silent", action="store_true"),
 758                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 759                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 760                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 761                                      help="Import into refs/heads/ , not refs/remotes"),
 762                 optparse.make_option("--max-changes", dest="maxChanges"),
 763                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 764                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 765                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 766                                      help="Only sync files that are included in the Perforce Client Spec")
 767         ]
 768         self.description = """Imports from Perforce into a git repository.\n
 769     example:
 770     //depot/my/project/ -- to import the current head
 771     //depot/my/project/@all -- to import everything
 772     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 773
 774     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 775
 776         self.usage += " //depot/path[@revRange]"
 777         self.silent = False
 778         self.createdBranches = Set()
 779         self.committedChanges = Set()
 780         self.branch = ""
 781         self.detectBranches = False
 782         self.detectLabels = False
 783         self.changesFile = ""
 784         self.syncWithOrigin = True
 785         self.verbose = False
 786         self.importIntoRemotes = True
 787         self.maxChanges = ""
 788         self.isWindows = (platform.system() == "Windows")
 789         self.keepRepoPath = False
 790         self.depotPaths = None
 791         self.p4BranchesInGit = []
 792         self.cloneExclude = []
 793         self.useClientSpec = False
 794         self.clientSpecDirs = []
 795
 796         if gitConfig("git-p4.syncFromOrigin") == "false":
 797             self.syncWithOrigin = False
 798
 799     def extractFilesFromCommit(self, commit):
 800         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 801                              for path in self.cloneExclude]
 802         files = []
 803         fnum = 0
 804         while commit.has_key("depotFile%s" % fnum):
 805             path =  commit["depotFile%s" % fnum]
 806
 807             if [p for p in self.cloneExclude
 808                 if path.startswith (p)]:
 809                 found = False
 810             else:
 811                 found = [p for p in self.depotPaths
 812                          if path.startswith (p)]
 813             if not found:
 814                 fnum = fnum + 1
 815                 continue
 816
 817             file = {}
 818             file["path"] = path
 819             file["rev"] = commit["rev%s" % fnum]
 820             file["action"] = commit["action%s" % fnum]
 821             file["type"] = commit["type%s" % fnum]
 822             files.append(file)
 823             fnum = fnum + 1
 824         return files
 825
 826     def stripRepoPath(self, path, prefixes):
 827         if self.keepRepoPath:
 828             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 829
 830         for p in prefixes:
 831             if path.startswith(p):
 832                 path = path[len(p):]
 833
 834         return path
 835
 836     def splitFilesIntoBranches(self, commit):
 837         branches = {}
 838         fnum = 0
 839         while commit.has_key("depotFile%s" % fnum):
 840             path =  commit["depotFile%s" % fnum]
 841             found = [p for p in self.depotPaths
 842                      if path.startswith (p)]
 843             if not found:
 844                 fnum = fnum + 1
 845                 continue
 846
 847             file = {}
 848             file["path"] = path
 849             file["rev"] = commit["rev%s" % fnum]
 850             file["action"] = commit["action%s" % fnum]
 851             file["type"] = commit["type%s" % fnum]
 852             fnum = fnum + 1
 853
 854             relPath = self.stripRepoPath(path, self.depotPaths)
 855
 856             for branch in self.knownBranches.keys():
 857
 858                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 859                 if relPath.startswith(branch + "/"):
 860                     if branch not in branches:
 861                         branches[branch] = []
 862                     branches[branch].append(file)
 863                     break
 864
 865         return branches
 866
 867     ## Should move this out, doesn't use SELF.
 868     def readP4Files(self, files):
 869         filesForCommit = []
 870         filesToRead = []
 871
 872         for f in files:
 873             includeFile = True
 874             for val in self.clientSpecDirs:
 875                 if f['path'].startswith(val[0]):
 876                     if val[1] <= 0:
 877                         includeFile = False
 878                     break
 879
 880             if includeFile:
 881                 filesForCommit.append(f)
 882                 if f['action'] != 'delete':
 883                     filesToRead.append(f)
 884
 885         filedata = []
 886         if len(filesToRead) > 0:
 887             filedata = p4CmdList('-x - print',
 888                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 889                                                   for f in filesToRead]),
 890                                  stdin_mode='w+')
 891
 892             if "p4ExitCode" in filedata[0]:
 893                 die("Problems executing p4. Error: [%d]."
 894                     % (filedata[0]['p4ExitCode']));
 895
 896         j = 0;
 897         contents = {}
 898         while j < len(filedata):
 899             stat = filedata[j]
 900             j += 1
 901             text = [];
 902             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 903                 text.append(filedata[j]['data'])
 904                 j += 1
 905             text = ''.join(text)
 906
 907             if not stat.has_key('depotFile'):
 908                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 909                 continue
 910
 911             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 912                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 913             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 914                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 915
 916             contents[stat['depotFile']] = text
 917
 918         for f in filesForCommit:
 919             path = f['path']
 920             if contents.has_key(path):
 921                 f['data'] = contents[path]
 922
 923         return filesForCommit
 924
 925     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 926         epoch = details["time"]
 927         author = details["user"]
 928
 929         if self.verbose:
 930             print "commit into %s" % branch
 931
 932         # start with reading files; if that fails, we should not
 933         # create a commit.
 934         new_files = []
 935         for f in files:
 936             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 937                 new_files.append (f)
 938             else:
 939                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 940         files = self.readP4Files(new_files)
 941
 942         self.gitStream.write("commit %s\n" % branch)
 943 #        gitStream.write("mark :%s\n" % details["change"])
 944         self.committedChanges.add(int(details["change"]))
 945         committer = ""
 946         if author not in self.users:
 947             self.getUserMapFromPerforceServer()
 948         if author in self.users:
 949             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 950         else:
 951             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 952
 953         self.gitStream.write("committer %s\n" % committer)
 954
 955         self.gitStream.write("data <<EOT\n")
 956         self.gitStream.write(details["desc"])
 957         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 958                              % (','.join (branchPrefixes), details["change"]))
 959         if len(details['options']) > 0:
 960             self.gitStream.write(": options = %s" % details['options'])
 961         self.gitStream.write("]\nEOT\n\n")
 962
 963         if len(parent) > 0:
 964             if self.verbose:
 965                 print "parent %s" % parent
 966             self.gitStream.write("from %s\n" % parent)
 967
 968         for file in files:
 969             if file["type"] == "apple":
 970                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 971                 continue
 972
 973             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 974             if file["action"] == "delete":
 975                 self.gitStream.write("D %s\n" % relPath)
 976             else:
 977                 data = file['data']
 978
 979                 mode = "644"
 980                 if isP4Exec(file["type"]):
 981                     mode = "755"
 982                 elif file["type"] == "symlink":
 983                     mode = "120000"
 984                     # p4 print on a symlink contains "target\n", so strip it off
 985                     data = data[:-1]
 986
 987                 if self.isWindows and file["type"].endswith("text"):
 988                     data = data.replace("\r\n", "\n")
 989
 990                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 991                 self.gitStream.write("data %s\n" % len(data))
 992                 self.gitStream.write(data)
 993                 self.gitStream.write("\n")
 994
 995         self.gitStream.write("\n")
 996
 997         change = int(details["change"])
 998
 999         if self.labels.has_key(change):
1000             label = self.labels[change]
1001             labelDetails = label[0]
1002             labelRevisions = label[1]
1003             if self.verbose:
1004                 print "Change %s is labelled %s" % (change, labelDetails)
1005
1006             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1007                                                     for p in branchPrefixes]))
1008
1009             if len(files) == len(labelRevisions):
1010
1011                 cleanedFiles = {}
1012                 for info in files:
1013                     if info["action"] == "delete":
1014                         continue
1015                     cleanedFiles[info["depotFile"]] = info["rev"]
1016
1017                 if cleanedFiles == labelRevisions:
1018                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1019                     self.gitStream.write("from %s\n" % branch)
1020
1021                     owner = labelDetails["Owner"]
1022                     tagger = ""
1023                     if author in self.users:
1024                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1025                     else:
1026                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1027                     self.gitStream.write("tagger %s\n" % tagger)
1028                     self.gitStream.write("data <<EOT\n")
1029                     self.gitStream.write(labelDetails["Description"])
1030                     self.gitStream.write("EOT\n\n")
1031
1032                 else:
1033                     if not self.silent:
1034                         print ("Tag %s does not match with change %s: files do not match."
1035                                % (labelDetails["label"], change))
1036
1037             else:
1038                 if not self.silent:
1039                     print ("Tag %s does not match with change %s: file count is different."
1040                            % (labelDetails["label"], change))
1041
1042     def getUserCacheFilename(self):
1043         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1044         return home + "/.gitp4-usercache.txt"
1045
1046     def getUserMapFromPerforceServer(self):
1047         if self.userMapFromPerforceServer:
1048             return
1049         self.users = {}
1050
1051         for output in p4CmdList("users"):
1052             if not output.has_key("User"):
1053                 continue
1054             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1055
1056
1057         s = ''
1058         for (key, val) in self.users.items():
1059             s += "%s\t%s\n" % (key, val)
1060
1061         open(self.getUserCacheFilename(), "wb").write(s)
1062         self.userMapFromPerforceServer = True
1063
1064     def loadUserMapFromCache(self):
1065         self.users = {}
1066         self.userMapFromPerforceServer = False
1067         try:
1068             cache = open(self.getUserCacheFilename(), "rb")
1069             lines = cache.readlines()
1070             cache.close()
1071             for line in lines:
1072                 entry = line.strip().split("\t")
1073                 self.users[entry[0]] = entry[1]
1074         except IOError:
1075             self.getUserMapFromPerforceServer()
1076
1077     def getLabels(self):
1078         self.labels = {}
1079
1080         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1081         if len(l) > 0 and not self.silent:
1082             print "Finding files belonging to labels in %s" % `self.depotPaths`
1083
1084         for output in l:
1085             label = output["label"]
1086             revisions = {}
1087             newestChange = 0
1088             if self.verbose:
1089                 print "Querying files for label %s" % label
1090             for file in p4CmdList("files "
1091                                   +  ' '.join (["%s...@%s" % (p, label)
1092                                                 for p in self.depotPaths])):
1093                 revisions[file["depotFile"]] = file["rev"]
1094                 change = int(file["change"])
1095                 if change > newestChange:
1096                     newestChange = change
1097
1098             self.labels[newestChange] = [output, revisions]
1099
1100         if self.verbose:
1101             print "Label changes: %s" % self.labels.keys()
1102
1103     def guessProjectName(self):
1104         for p in self.depotPaths:
1105             if p.endswith("/"):
1106                 p = p[:-1]
1107             p = p[p.strip().rfind("/") + 1:]
1108             if not p.endswith("/"):
1109                p += "/"
1110             return p
1111
1112     def getBranchMapping(self):
1113         lostAndFoundBranches = set()
1114
1115         for info in p4CmdList("branches"):
1116             details = p4Cmd("branch -o %s" % info["branch"])
1117             viewIdx = 0
1118             while details.has_key("View%s" % viewIdx):
1119                 paths = details["View%s" % viewIdx].split(" ")
1120                 viewIdx = viewIdx + 1
1121                 # require standard //depot/foo/... //depot/bar/... mapping
1122                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1123                     continue
1124                 source = paths[0]
1125                 destination = paths[1]
1126                 ## HACK
1127                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1128                     source = source[len(self.depotPaths[0]):-4]
1129                     destination = destination[len(self.depotPaths[0]):-4]
1130
1131                     if destination in self.knownBranches:
1132                         if not self.silent:
1133                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1134                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1135                         continue
1136
1137                     self.knownBranches[destination] = source
1138
1139                     lostAndFoundBranches.discard(destination)
1140
1141                     if source not in self.knownBranches:
1142                         lostAndFoundBranches.add(source)
1143
1144
1145         for branch in lostAndFoundBranches:
1146             self.knownBranches[branch] = branch
1147
1148     def getBranchMappingFromGitBranches(self):
1149         branches = p4BranchesInGit(self.importIntoRemotes)
1150         for branch in branches.keys():
1151             if branch == "master":
1152                 branch = "main"
1153             else:
1154                 branch = branch[len(self.projectName):]
1155             self.knownBranches[branch] = branch
1156
1157     def listExistingP4GitBranches(self):
1158         # branches holds mapping from name to commit
1159         branches = p4BranchesInGit(self.importIntoRemotes)
1160         self.p4BranchesInGit = branches.keys()
1161         for branch in branches.keys():
1162             self.initialParents[self.refPrefix + branch] = branches[branch]
1163
1164     def updateOptionDict(self, d):
1165         option_keys = {}
1166         if self.keepRepoPath:
1167             option_keys['keepRepoPath'] = 1
1168
1169         d["options"] = ' '.join(sorted(option_keys.keys()))
1170
1171     def readOptions(self, d):
1172         self.keepRepoPath = (d.has_key('options')
1173                              and ('keepRepoPath' in d['options']))
1174
1175     def gitRefForBranch(self, branch):
1176         if branch == "main":
1177             return self.refPrefix + "master"
1178
1179         if len(branch) <= 0:
1180             return branch
1181
1182         return self.refPrefix + self.projectName + branch
1183
1184     def gitCommitByP4Change(self, ref, change):
1185         if self.verbose:
1186             print "looking in ref " + ref + " for change %s using bisect..." % change
1187
1188         earliestCommit = ""
1189         latestCommit = parseRevision(ref)
1190
1191         while True:
1192             if self.verbose:
1193                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1194             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1195             if len(next) == 0:
1196                 if self.verbose:
1197                     print "argh"
1198                 return ""
1199             log = extractLogMessageFromGitCommit(next)
1200             settings = extractSettingsGitLog(log)
1201             currentChange = int(settings['change'])
1202             if self.verbose:
1203                 print "current change %s" % currentChange
1204
1205             if currentChange == change:
1206                 if self.verbose:
1207                     print "found %s" % next
1208                 return next
1209
1210             if currentChange < change:
1211                 earliestCommit = "^%s" % next
1212             else:
1213                 latestCommit = "%s" % next
1214
1215         return ""
1216
1217     def importNewBranch(self, branch, maxChange):
1218         # make fast-import flush all changes to disk and update the refs using the checkpoint
1219         # command so that we can try to find the branch parent in the git history
1220         self.gitStream.write("checkpoint\n\n");
1221         self.gitStream.flush();
1222         branchPrefix = self.depotPaths[0] + branch + "/"
1223         range = "@1,%s" % maxChange
1224         #print "prefix" + branchPrefix
1225         changes = p4ChangesForPaths([branchPrefix], range)
1226         if len(changes) <= 0:
1227             return False
1228         firstChange = changes[0]
1229         #print "first change in branch: %s" % firstChange
1230         sourceBranch = self.knownBranches[branch]
1231         sourceDepotPath = self.depotPaths[0] + sourceBranch
1232         sourceRef = self.gitRefForBranch(sourceBranch)
1233         #print "source " + sourceBranch
1234
1235         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1236         #print "branch parent: %s" % branchParentChange
1237         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1238         if len(gitParent) > 0:
1239             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1240             #print "parent git commit: %s" % gitParent
1241
1242         self.importChanges(changes)
1243         return True
1244
1245     def importChanges(self, changes):
1246         cnt = 1
1247         for change in changes:
1248             description = p4Cmd("describe %s" % change)
1249             self.updateOptionDict(description)
1250
1251             if not self.silent:
1252                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1253                 sys.stdout.flush()
1254             cnt = cnt + 1
1255
1256             try:
1257                 if self.detectBranches:
1258                     branches = self.splitFilesIntoBranches(description)
1259                     for branch in branches.keys():
1260                         ## HACK  --hwn
1261                         branchPrefix = self.depotPaths[0] + branch + "/"
1262
1263                         parent = ""
1264
1265                         filesForCommit = branches[branch]
1266
1267                         if self.verbose:
1268                             print "branch is %s" % branch
1269
1270                         self.updatedBranches.add(branch)
1271
1272                         if branch not in self.createdBranches:
1273                             self.createdBranches.add(branch)
1274                             parent = self.knownBranches[branch]
1275                             if parent == branch:
1276                                 parent = ""
1277                             else:
1278                                 fullBranch = self.projectName + branch
1279                                 if fullBranch not in self.p4BranchesInGit:
1280                                     if not self.silent:
1281                                         print("\n    Importing new branch %s" % fullBranch);
1282                                     if self.importNewBranch(branch, change - 1):
1283                                         parent = ""
1284                                         self.p4BranchesInGit.append(fullBranch)
1285                                     if not self.silent:
1286                                         print("\n    Resuming with change %s" % change);
1287
1288                                 if self.verbose:
1289                                     print "parent determined through known branches: %s" % parent
1290
1291                         branch = self.gitRefForBranch(branch)
1292                         parent = self.gitRefForBranch(parent)
1293
1294                         if self.verbose:
1295                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1296
1297                         if len(parent) == 0 and branch in self.initialParents:
1298                             parent = self.initialParents[branch]
1299                             del self.initialParents[branch]
1300
1301                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1302                 else:
1303                     files = self.extractFilesFromCommit(description)
1304                     self.commit(description, files, self.branch, self.depotPaths,
1305                                 self.initialParent)
1306                     self.initialParent = ""
1307             except IOError:
1308                 print self.gitError.read()
1309                 sys.exit(1)
1310
1311     def importHeadRevision(self, revision):
1312         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1313
1314         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1315         details["desc"] = ("Initial import of %s from the state at revision %s"
1316                            % (' '.join(self.depotPaths), revision))
1317         details["change"] = revision
1318         newestRevision = 0
1319
1320         fileCnt = 0
1321         for info in p4CmdList("files "
1322                               +  ' '.join(["%s...%s"
1323                                            % (p, revision)
1324                                            for p in self.depotPaths])):
1325
1326             if info['code'] == 'error':
1327                 sys.stderr.write("p4 returned an error: %s\n"
1328                                  % info['data'])
1329                 sys.exit(1)
1330
1331
1332             change = int(info["change"])
1333             if change > newestRevision:
1334                 newestRevision = change
1335
1336             if info["action"] == "delete":
1337                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1338                 #fileCnt = fileCnt + 1
1339                 continue
1340
1341             for prop in ["depotFile", "rev", "action", "type" ]:
1342                 details["%s%s" % (prop, fileCnt)] = info[prop]
1343
1344             fileCnt = fileCnt + 1
1345
1346         details["change"] = newestRevision
1347         self.updateOptionDict(details)
1348         try:
1349             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1350         except IOError:
1351             print "IO error with git fast-import. Is your git version recent enough?"
1352             print self.gitError.read()
1353
1354
1355     def getClientSpec(self):
1356         specList = p4CmdList( "client -o" )
1357         temp = {}
1358         for entry in specList:
1359             for k,v in entry.iteritems():
1360                 if k.startswith("View"):
1361                     if v.startswith('"'):
1362                         start = 1
1363                     else:
1364                         start = 0
1365                     index = v.find("...")
1366                     v = v[start:index]
1367                     if v.startswith("-"):
1368                         v = v[1:]
1369                         temp[v] = -len(v)
1370                     else:
1371                         temp[v] = len(v)
1372         self.clientSpecDirs = temp.items()
1373         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1374
1375     def run(self, args):
1376         self.depotPaths = []
1377         self.changeRange = ""
1378         self.initialParent = ""
1379         self.previousDepotPaths = []
1380
1381         # map from branch depot path to parent branch
1382         self.knownBranches = {}
1383         self.initialParents = {}
1384         self.hasOrigin = originP4BranchesExist()
1385         if not self.syncWithOrigin:
1386             self.hasOrigin = False
1387
1388         if self.importIntoRemotes:
1389             self.refPrefix = "refs/remotes/p4/"
1390         else:
1391             self.refPrefix = "refs/heads/p4/"
1392
1393         if self.syncWithOrigin and self.hasOrigin:
1394             if not self.silent:
1395                 print "Syncing with origin first by calling git fetch origin"
1396             system("git fetch origin")
1397
1398         if len(self.branch) == 0:
1399             self.branch = self.refPrefix + "master"
1400             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1401                 system("git update-ref %s refs/heads/p4" % self.branch)
1402                 system("git branch -D p4");
1403             # create it /after/ importing, when master exists
1404             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1405                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1406
1407         if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1408             self.getClientSpec()
1409
1410         # TODO: should always look at previous commits,
1411         # merge with previous imports, if possible.
1412         if args == []:
1413             if self.hasOrigin:
1414                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1415             self.listExistingP4GitBranches()
1416
1417             if len(self.p4BranchesInGit) > 1:
1418                 if not self.silent:
1419                     print "Importing from/into multiple branches"
1420                 self.detectBranches = True
1421
1422             if self.verbose:
1423                 print "branches: %s" % self.p4BranchesInGit
1424
1425             p4Change = 0
1426             for branch in self.p4BranchesInGit:
1427                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1428
1429                 settings = extractSettingsGitLog(logMsg)
1430
1431                 self.readOptions(settings)
1432                 if (settings.has_key('depot-paths')
1433                     and settings.has_key ('change')):
1434                     change = int(settings['change']) + 1
1435                     p4Change = max(p4Change, change)
1436
1437                     depotPaths = sorted(settings['depot-paths'])
1438                     if self.previousDepotPaths == []:
1439                         self.previousDepotPaths = depotPaths
1440                     else:
1441                         paths = []
1442                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1443                             for i in range(0, min(len(cur), len(prev))):
1444                                 if cur[i] <> prev[i]:
1445                                     i = i - 1
1446                                     break
1447
1448                             paths.append (cur[:i + 1])
1449
1450                         self.previousDepotPaths = paths
1451
1452             if p4Change > 0:
1453                 self.depotPaths = sorted(self.previousDepotPaths)
1454                 self.changeRange = "@%s,#head" % p4Change
1455                 if not self.detectBranches:
1456                     self.initialParent = parseRevision(self.branch)
1457                 if not self.silent and not self.detectBranches:
1458                     print "Performing incremental import into %s git branch" % self.branch
1459
1460         if not self.branch.startswith("refs/"):
1461             self.branch = "refs/heads/" + self.branch
1462
1463         if len(args) == 0 and self.depotPaths:
1464             if not self.silent:
1465                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1466         else:
1467             if self.depotPaths and self.depotPaths != args:
1468                 print ("previous import used depot path %s and now %s was specified. "
1469                        "This doesn't work!" % (' '.join (self.depotPaths),
1470                                                ' '.join (args)))
1471                 sys.exit(1)
1472
1473             self.depotPaths = sorted(args)
1474
1475         revision = ""
1476         self.users = {}
1477
1478         newPaths = []
1479         for p in self.depotPaths:
1480             if p.find("@") != -1:
1481                 atIdx = p.index("@")
1482                 self.changeRange = p[atIdx:]
1483                 if self.changeRange == "@all":
1484                     self.changeRange = ""
1485                 elif ',' not in self.changeRange:
1486                     revision = self.changeRange
1487                     self.changeRange = ""
1488                 p = p[:atIdx]
1489             elif p.find("#") != -1:
1490                 hashIdx = p.index("#")
1491                 revision = p[hashIdx:]
1492                 p = p[:hashIdx]
1493             elif self.previousDepotPaths == []:
1494                 revision = "#head"
1495
1496             p = re.sub ("\.\.\.$", "", p)
1497             if not p.endswith("/"):
1498                 p += "/"
1499
1500             newPaths.append(p)
1501
1502         self.depotPaths = newPaths
1503
1504
1505         self.loadUserMapFromCache()
1506         self.labels = {}
1507         if self.detectLabels:
1508             self.getLabels();
1509
1510         if self.detectBranches:
1511             ## FIXME - what's a P4 projectName ?
1512             self.projectName = self.guessProjectName()
1513
1514             if self.hasOrigin:
1515                 self.getBranchMappingFromGitBranches()
1516             else:
1517                 self.getBranchMapping()
1518             if self.verbose:
1519                 print "p4-git branches: %s" % self.p4BranchesInGit
1520                 print "initial parents: %s" % self.initialParents
1521             for b in self.p4BranchesInGit:
1522                 if b != "master":
1523
1524                     ## FIXME
1525                     b = b[len(self.projectName):]
1526                 self.createdBranches.add(b)
1527
1528         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1529
1530         importProcess = subprocess.Popen(["git", "fast-import"],
1531                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1532                                          stderr=subprocess.PIPE);
1533         self.gitOutput = importProcess.stdout
1534         self.gitStream = importProcess.stdin
1535         self.gitError = importProcess.stderr
1536
1537         if revision:
1538             self.importHeadRevision(revision)
1539         else:
1540             changes = []
1541
1542             if len(self.changesFile) > 0:
1543                 output = open(self.changesFile).readlines()
1544                 changeSet = Set()
1545                 for line in output:
1546                     changeSet.add(int(line))
1547
1548                 for change in changeSet:
1549                     changes.append(change)
1550
1551                 changes.sort()
1552             else:
1553                 if self.verbose:
1554                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1555                                                               self.changeRange)
1556                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1557
1558                 if len(self.maxChanges) > 0:
1559                     changes = changes[:min(int(self.maxChanges), len(changes))]
1560
1561             if len(changes) == 0:
1562                 if not self.silent:
1563                     print "No changes to import!"
1564                 return True
1565
1566             if not self.silent and not self.detectBranches:
1567                 print "Import destination: %s" % self.branch
1568
1569             self.updatedBranches = set()
1570
1571             self.importChanges(changes)
1572
1573             if not self.silent:
1574                 print ""
1575                 if len(self.updatedBranches) > 0:
1576                     sys.stdout.write("Updated branches: ")
1577                     for b in self.updatedBranches:
1578                         sys.stdout.write("%s " % b)
1579                     sys.stdout.write("\n")
1580
1581         self.gitStream.close()
1582         if importProcess.wait() != 0:
1583             die("fast-import failed: %s" % self.gitError.read())
1584         self.gitOutput.close()
1585         self.gitError.close()
1586
1587         return True
1588
1589 class P4Rebase(Command):
1590     def __init__(self):
1591         Command.__init__(self)
1592         self.options = [ ]
1593         self.description = ("Fetches the latest revision from perforce and "
1594                             + "rebases the current work (branch) against it")
1595         self.verbose = False
1596
1597     def run(self, args):
1598         sync = P4Sync()
1599         sync.run([])
1600
1601         return self.rebase()
1602
1603     def rebase(self):
1604         if os.system("git update-index --refresh") != 0:
1605             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1606         if len(read_pipe("git diff-index HEAD --")) > 0:
1607             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1608
1609         [upstream, settings] = findUpstreamBranchPoint()
1610         if len(upstream) == 0:
1611             die("Cannot find upstream branchpoint for rebase")
1612
1613         # the branchpoint may be p4/foo~3, so strip off the parent
1614         upstream = re.sub("~[0-9]+$", "", upstream)
1615
1616         print "Rebasing the current branch onto %s" % upstream
1617         oldHead = read_pipe("git rev-parse HEAD").strip()
1618         system("git rebase %s" % upstream)
1619         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1620         return True
1621
1622 class P4Clone(P4Sync):
1623     def __init__(self):
1624         P4Sync.__init__(self)
1625         self.description = "Creates a new git repository and imports from Perforce into it"
1626         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1627         self.options += [
1628             optparse.make_option("--destination", dest="cloneDestination",
1629                                  action='store', default=None,
1630                                  help="where to leave result of the clone"),
1631             optparse.make_option("-/", dest="cloneExclude",
1632                                  action="append", type="string",
1633                                  help="exclude depot path")
1634         ]
1635         self.cloneDestination = None
1636         self.needsGit = False
1637
1638     # This is required for the "append" cloneExclude action
1639     def ensure_value(self, attr, value):
1640         if not hasattr(self, attr) or getattr(self, attr) is None:
1641             setattr(self, attr, value)
1642         return getattr(self, attr)
1643
1644     def defaultDestination(self, args):
1645         ## TODO: use common prefix of args?
1646         depotPath = args[0]
1647         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1648         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1649         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1650         depotDir = re.sub(r"/$", "", depotDir)
1651         return os.path.split(depotDir)[1]
1652
1653     def run(self, args):
1654         if len(args) < 1:
1655             return False
1656
1657         if self.keepRepoPath and not self.cloneDestination:
1658             sys.stderr.write("Must specify destination for --keep-path\n")
1659             sys.exit(1)
1660
1661         depotPaths = args
1662
1663         if not self.cloneDestination and len(depotPaths) > 1:
1664             self.cloneDestination = depotPaths[-1]
1665             depotPaths = depotPaths[:-1]
1666
1667         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1668         for p in depotPaths:
1669             if not p.startswith("//"):
1670                 return False
1671
1672         if not self.cloneDestination:
1673             self.cloneDestination = self.defaultDestination(args)
1674
1675         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1676         if not os.path.exists(self.cloneDestination):
1677             os.makedirs(self.cloneDestination)
1678         chdir(self.cloneDestination)
1679         system("git init")
1680         self.gitdir = os.getcwd() + "/.git"
1681         if not P4Sync.run(self, depotPaths):
1682             return False
1683         if self.branch != "master":
1684             if gitBranchExists("refs/remotes/p4/master"):
1685                 system("git branch master refs/remotes/p4/master")
1686                 system("git checkout -f")
1687             else:
1688                 print "Could not detect main branch. No checkout/master branch created."
1689
1690         return True
1691
1692 class P4Branches(Command):
1693     def __init__(self):
1694         Command.__init__(self)
1695         self.options = [ ]
1696         self.description = ("Shows the git branches that hold imports and their "
1697                             + "corresponding perforce depot paths")
1698         self.verbose = False
1699
1700     def run(self, args):
1701         if originP4BranchesExist():
1702             createOrUpdateBranchesFromOrigin()
1703
1704         cmdline = "git rev-parse --symbolic "
1705         cmdline += " --remotes"
1706
1707         for line in read_pipe_lines(cmdline):
1708             line = line.strip()
1709
1710             if not line.startswith('p4/') or line == "p4/HEAD":
1711                 continue
1712             branch = line
1713
1714             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1715             settings = extractSettingsGitLog(log)
1716
1717             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1718         return True
1719
1720 class HelpFormatter(optparse.IndentedHelpFormatter):
1721     def __init__(self):
1722         optparse.IndentedHelpFormatter.__init__(self)
1723
1724     def format_description(self, description):
1725         if description:
1726             return description + "\n"
1727         else:
1728             return ""
1729
1730 def printUsage(commands):
1731     print "usage: %s <command> [options]" % sys.argv[0]
1732     print ""
1733     print "valid commands: %s" % ", ".join(commands)
1734     print ""
1735     print "Try %s <command> --help for command specific help." % sys.argv[0]
1736     print ""
1737
1738 commands = {
1739     "debug" : P4Debug,
1740     "submit" : P4Submit,
1741     "commit" : P4Submit,
1742     "sync" : P4Sync,
1743     "rebase" : P4Rebase,
1744     "clone" : P4Clone,
1745     "rollback" : P4RollBack,
1746     "branches" : P4Branches
1747 }
1748
1749
1750 def main():
1751     if len(sys.argv[1:]) == 0:
1752         printUsage(commands.keys())
1753         sys.exit(2)
1754
1755     cmd = ""
1756     cmdName = sys.argv[1]
1757     try:
1758         klass = commands[cmdName]
1759         cmd = klass()
1760     except KeyError:
1761         print "unknown command %s" % cmdName
1762         print ""
1763         printUsage(commands.keys())
1764         sys.exit(2)
1765
1766     options = cmd.options
1767     cmd.gitdir = os.environ.get("GIT_DIR", None)
1768
1769     args = sys.argv[2:]
1770
1771     if len(options) > 0:
1772         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1773
1774         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1775                                        options,
1776                                        description = cmd.description,
1777                                        formatter = HelpFormatter())
1778
1779         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1780     global verbose
1781     verbose = cmd.verbose
1782     if cmd.needsGit:
1783         if cmd.gitdir == None:
1784             cmd.gitdir = os.path.abspath(".git")
1785             if not isValidGitDir(cmd.gitdir):
1786                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1787                 if os.path.exists(cmd.gitdir):
1788                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1789                     if len(cdup) > 0:
1790                         chdir(cdup);
1791
1792         if not isValidGitDir(cmd.gitdir):
1793             if isValidGitDir(cmd.gitdir + "/.git"):
1794                 cmd.gitdir += "/.git"
1795             else:
1796                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1797
1798         os.environ["GIT_DIR"] = cmd.gitdir
1799
1800     if not cmd.run(args):
1801         parser.print_help()
1802
1803
1804 if __name__ == '__main__':
1805     main()