contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s %s" % ("p4", cmd)
  28     if verbose:
  29         print real_cmd
  30     return real_cmd
  31
  32 def die(msg):
  33     if verbose:
  34         raise Exception(msg)
  35     else:
  36         sys.stderr.write(msg + "\n")
  37         sys.exit(1)
  38
  39 def write_pipe(c, str):
  40     if verbose:
  41         sys.stderr.write('Writing pipe: %s\n' % c)
  42
  43     pipe = os.popen(c, 'w')
  44     val = pipe.write(str)
  45     if pipe.close():
  46         die('Command failed: %s' % c)
  47
  48     return val
  49
  50 def read_pipe(c, ignore_error=False):
  51     if verbose:
  52         sys.stderr.write('Reading pipe: %s\n' % c)
  53
  54     pipe = os.popen(c, 'rb')
  55     val = pipe.read()
  56     if pipe.close() and not ignore_error:
  57         die('Command failed: %s' % c)
  58
  59     return val
  60
  61
  62 def read_pipe_lines(c):
  63     if verbose:
  64         sys.stderr.write('Reading pipe: %s\n' % c)
  65     ## todo: check return status
  66     pipe = os.popen(c, 'rb')
  67     val = pipe.readlines()
  68     if pipe.close():
  69         die('Command failed: %s' % c)
  70
  71     return val
  72
  73 def p4_read_pipe_lines(c):
  74     """Specifically invoke p4 on the command supplied. """
  75     real_cmd = p4_build_cmd(c)
  76     return read_pipe_lines(real_cmd)
  77
  78 def system(cmd):
  79     if verbose:
  80         sys.stderr.write("executing %s\n" % cmd)
  81     if os.system(cmd) != 0:
  82         die("command failed: %s" % cmd)
  83
  84 def p4_system(cmd):
  85     """Specifically invoke p4 as the system command. """
  86     real_cmd = p4_build_cmd(cmd)
  87     return system(real_cmd)
  88
  89 def isP4Exec(kind):
  90     """Determine if a Perforce 'kind' should have execute permission
  91
  92     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  93     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  94     a plus sign, it is also executable"""
  95     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  96
  97 def setP4ExecBit(file, mode):
  98     # Reopens an already open file and changes the execute bit to match
  99     # the execute bit setting in the passed in mode.
 100
 101     p4Type = "+x"
 102
 103     if not isModeExec(mode):
 104         p4Type = getP4OpenedType(file)
 105         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 106         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 107         if p4Type[-1] == "+":
 108             p4Type = p4Type[0:-1]
 109
 110     p4_system("reopen -t %s %s" % (p4Type, file))
 111
 112 def getP4OpenedType(file):
 113     # Returns the perforce file type for the given file.
 114
 115     result = read_pipe("p4 opened %s" % file)
 116     match = re.match(".*\((.+)\)\r?$", result)
 117     if match:
 118         return match.group(1)
 119     else:
 120         die("Could not determine file type for %s (result: '%s')" % (file, result))
 121
 122 def diffTreePattern():
 123     # This is a simple generator for the diff tree regex pattern. This could be
 124     # a class variable if this and parseDiffTreeEntry were a part of a class.
 125     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 126     while True:
 127         yield pattern
 128
 129 def parseDiffTreeEntry(entry):
 130     """Parses a single diff tree entry into its component elements.
 131
 132     See git-diff-tree(1) manpage for details about the format of the diff
 133     output. This method returns a dictionary with the following elements:
 134
 135     src_mode - The mode of the source file
 136     dst_mode - The mode of the destination file
 137     src_sha1 - The sha1 for the source file
 138     dst_sha1 - The sha1 fr the destination file
 139     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 140     status_score - The score for the status (applicable for 'C' and 'R'
 141                    statuses). This is None if there is no score.
 142     src - The path for the source file.
 143     dst - The path for the destination file. This is only present for
 144           copy or renames. If it is not present, this is None.
 145
 146     If the pattern is not matched, None is returned."""
 147
 148     match = diffTreePattern().next().match(entry)
 149     if match:
 150         return {
 151             'src_mode': match.group(1),
 152             'dst_mode': match.group(2),
 153             'src_sha1': match.group(3),
 154             'dst_sha1': match.group(4),
 155             'status': match.group(5),
 156             'status_score': match.group(6),
 157             'src': match.group(7),
 158             'dst': match.group(10)
 159         }
 160     return None
 161
 162 def isModeExec(mode):
 163     # Returns True if the given git mode represents an executable file,
 164     # otherwise False.
 165     return mode[-3:] == "755"
 166
 167 def isModeExecChanged(src_mode, dst_mode):
 168     return isModeExec(src_mode) != isModeExec(dst_mode)
 169
 170 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 171     cmd = p4_build_cmd("-G %s" % (cmd))
 172     if verbose:
 173         sys.stderr.write("Opening pipe: %s\n" % cmd)
 174
 175     # Use a temporary file to avoid deadlocks without
 176     # subprocess.communicate(), which would put another copy
 177     # of stdout into memory.
 178     stdin_file = None
 179     if stdin is not None:
 180         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 181         stdin_file.write(stdin)
 182         stdin_file.flush()
 183         stdin_file.seek(0)
 184
 185     p4 = subprocess.Popen(cmd, shell=True,
 186                           stdin=stdin_file,
 187                           stdout=subprocess.PIPE)
 188
 189     result = []
 190     try:
 191         while True:
 192             entry = marshal.load(p4.stdout)
 193             result.append(entry)
 194     except EOFError:
 195         pass
 196     exitCode = p4.wait()
 197     if exitCode != 0:
 198         entry = {}
 199         entry["p4ExitCode"] = exitCode
 200         result.append(entry)
 201
 202     return result
 203
 204 def p4Cmd(cmd):
 205     list = p4CmdList(cmd)
 206     result = {}
 207     for entry in list:
 208         result.update(entry)
 209     return result;
 210
 211 def p4Where(depotPath):
 212     if not depotPath.endswith("/"):
 213         depotPath += "/"
 214     output = p4Cmd("where %s..." % depotPath)
 215     if output["code"] == "error":
 216         return ""
 217     clientPath = ""
 218     if "path" in output:
 219         clientPath = output.get("path")
 220     elif "data" in output:
 221         data = output.get("data")
 222         lastSpace = data.rfind(" ")
 223         clientPath = data[lastSpace + 1:]
 224
 225     if clientPath.endswith("..."):
 226         clientPath = clientPath[:-3]
 227     return clientPath
 228
 229 def currentGitBranch():
 230     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 231
 232 def isValidGitDir(path):
 233     if (os.path.exists(path + "/HEAD")
 234         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 235         return True;
 236     return False
 237
 238 def parseRevision(ref):
 239     return read_pipe("git rev-parse %s" % ref).strip()
 240
 241 def extractLogMessageFromGitCommit(commit):
 242     logMessage = ""
 243
 244     ## fixme: title is first line of commit, not 1st paragraph.
 245     foundTitle = False
 246     for log in read_pipe_lines("git cat-file commit %s" % commit):
 247        if not foundTitle:
 248            if len(log) == 1:
 249                foundTitle = True
 250            continue
 251
 252        logMessage += log
 253     return logMessage
 254
 255 def extractSettingsGitLog(log):
 256     values = {}
 257     for line in log.split("\n"):
 258         line = line.strip()
 259         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 260         if not m:
 261             continue
 262
 263         assignments = m.group(1).split (':')
 264         for a in assignments:
 265             vals = a.split ('=')
 266             key = vals[0].strip()
 267             val = ('='.join (vals[1:])).strip()
 268             if val.endswith ('\"') and val.startswith('"'):
 269                 val = val[1:-1]
 270
 271             values[key] = val
 272
 273     paths = values.get("depot-paths")
 274     if not paths:
 275         paths = values.get("depot-path")
 276     if paths:
 277         values['depot-paths'] = paths.split(',')
 278     return values
 279
 280 def gitBranchExists(branch):
 281     proc = subprocess.Popen(["git", "rev-parse", branch],
 282                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 283     return proc.wait() == 0;
 284
 285 def gitConfig(key):
 286     return read_pipe("git config %s" % key, ignore_error=True).strip()
 287
 288 def p4BranchesInGit(branchesAreInRemotes = True):
 289     branches = {}
 290
 291     cmdline = "git rev-parse --symbolic "
 292     if branchesAreInRemotes:
 293         cmdline += " --remotes"
 294     else:
 295         cmdline += " --branches"
 296
 297     for line in read_pipe_lines(cmdline):
 298         line = line.strip()
 299
 300         ## only import to p4/
 301         if not line.startswith('p4/') or line == "p4/HEAD":
 302             continue
 303         branch = line
 304
 305         # strip off p4
 306         branch = re.sub ("^p4/", "", line)
 307
 308         branches[branch] = parseRevision(line)
 309     return branches
 310
 311 def findUpstreamBranchPoint(head = "HEAD"):
 312     branches = p4BranchesInGit()
 313     # map from depot-path to branch name
 314     branchByDepotPath = {}
 315     for branch in branches.keys():
 316         tip = branches[branch]
 317         log = extractLogMessageFromGitCommit(tip)
 318         settings = extractSettingsGitLog(log)
 319         if settings.has_key("depot-paths"):
 320             paths = ",".join(settings["depot-paths"])
 321             branchByDepotPath[paths] = "remotes/p4/" + branch
 322
 323     settings = None
 324     parent = 0
 325     while parent < 65535:
 326         commit = head + "~%s" % parent
 327         log = extractLogMessageFromGitCommit(commit)
 328         settings = extractSettingsGitLog(log)
 329         if settings.has_key("depot-paths"):
 330             paths = ",".join(settings["depot-paths"])
 331             if branchByDepotPath.has_key(paths):
 332                 return [branchByDepotPath[paths], settings]
 333
 334         parent = parent + 1
 335
 336     return ["", settings]
 337
 338 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 339     if not silent:
 340         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 341                % localRefPrefix)
 342
 343     originPrefix = "origin/p4/"
 344
 345     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 346         line = line.strip()
 347         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 348             continue
 349
 350         headName = line[len(originPrefix):]
 351         remoteHead = localRefPrefix + headName
 352         originHead = line
 353
 354         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 355         if (not original.has_key('depot-paths')
 356             or not original.has_key('change')):
 357             continue
 358
 359         update = False
 360         if not gitBranchExists(remoteHead):
 361             if verbose:
 362                 print "creating %s" % remoteHead
 363             update = True
 364         else:
 365             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 366             if settings.has_key('change') > 0:
 367                 if settings['depot-paths'] == original['depot-paths']:
 368                     originP4Change = int(original['change'])
 369                     p4Change = int(settings['change'])
 370                     if originP4Change > p4Change:
 371                         print ("%s (%s) is newer than %s (%s). "
 372                                "Updating p4 branch from origin."
 373                                % (originHead, originP4Change,
 374                                   remoteHead, p4Change))
 375                         update = True
 376                 else:
 377                     print ("Ignoring: %s was imported from %s while "
 378                            "%s was imported from %s"
 379                            % (originHead, ','.join(original['depot-paths']),
 380                               remoteHead, ','.join(settings['depot-paths'])))
 381
 382         if update:
 383             system("git update-ref %s %s" % (remoteHead, originHead))
 384
 385 def originP4BranchesExist():
 386         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 387
 388 def p4ChangesForPaths(depotPaths, changeRange):
 389     assert depotPaths
 390     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 391                                                         for p in depotPaths]))
 392
 393     changes = []
 394     for line in output:
 395         changeNum = line.split(" ")[1]
 396         changes.append(int(changeNum))
 397
 398     changes.sort()
 399     return changes
 400
 401 class Command:
 402     def __init__(self):
 403         self.usage = "usage: %prog [options]"
 404         self.needsGit = True
 405
 406 class P4Debug(Command):
 407     def __init__(self):
 408         Command.__init__(self)
 409         self.options = [
 410             optparse.make_option("--verbose", dest="verbose", action="store_true",
 411                                  default=False),
 412             ]
 413         self.description = "A tool to debug the output of p4 -G."
 414         self.needsGit = False
 415         self.verbose = False
 416
 417     def run(self, args):
 418         j = 0
 419         for output in p4CmdList(" ".join(args)):
 420             print 'Element: %d' % j
 421             j += 1
 422             print output
 423         return True
 424
 425 class P4RollBack(Command):
 426     def __init__(self):
 427         Command.__init__(self)
 428         self.options = [
 429             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 430             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 431         ]
 432         self.description = "A tool to debug the multi-branch import. Don't use :)"
 433         self.verbose = False
 434         self.rollbackLocalBranches = False
 435
 436     def run(self, args):
 437         if len(args) != 1:
 438             return False
 439         maxChange = int(args[0])
 440
 441         if "p4ExitCode" in p4Cmd("changes -m 1"):
 442             die("Problems executing p4");
 443
 444         if self.rollbackLocalBranches:
 445             refPrefix = "refs/heads/"
 446             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 447         else:
 448             refPrefix = "refs/remotes/"
 449             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 450
 451         for line in lines:
 452             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 453                 line = line.strip()
 454                 ref = refPrefix + line
 455                 log = extractLogMessageFromGitCommit(ref)
 456                 settings = extractSettingsGitLog(log)
 457
 458                 depotPaths = settings['depot-paths']
 459                 change = settings['change']
 460
 461                 changed = False
 462
 463                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 464                                                            for p in depotPaths]))) == 0:
 465                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 466                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 467                     continue
 468
 469                 while change and int(change) > maxChange:
 470                     changed = True
 471                     if self.verbose:
 472                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 473                     system("git update-ref %s \"%s^\"" % (ref, ref))
 474                     log = extractLogMessageFromGitCommit(ref)
 475                     settings =  extractSettingsGitLog(log)
 476
 477
 478                     depotPaths = settings['depot-paths']
 479                     change = settings['change']
 480
 481                 if changed:
 482                     print "%s rewound to %s" % (ref, change)
 483
 484         return True
 485
 486 class P4Submit(Command):
 487     def __init__(self):
 488         Command.__init__(self)
 489         self.options = [
 490                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 491                 optparse.make_option("--origin", dest="origin"),
 492                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 493         ]
 494         self.description = "Submit changes from git to the perforce depot."
 495         self.usage += " [name of git branch to submit into perforce depot]"
 496         self.interactive = True
 497         self.origin = ""
 498         self.detectRename = False
 499         self.verbose = False
 500         self.isWindows = (platform.system() == "Windows")
 501
 502     def check(self):
 503         if len(p4CmdList("opened ...")) > 0:
 504             die("You have files opened with perforce! Close them before starting the sync.")
 505
 506     # replaces everything between 'Description:' and the next P4 submit template field with the
 507     # commit message
 508     def prepareLogMessage(self, template, message):
 509         result = ""
 510
 511         inDescriptionSection = False
 512
 513         for line in template.split("\n"):
 514             if line.startswith("#"):
 515                 result += line + "\n"
 516                 continue
 517
 518             if inDescriptionSection:
 519                 if line.startswith("Files:"):
 520                     inDescriptionSection = False
 521                 else:
 522                     continue
 523             else:
 524                 if line.startswith("Description:"):
 525                     inDescriptionSection = True
 526                     line += "\n"
 527                     for messageLine in message.split("\n"):
 528                         line += "\t" + messageLine + "\n"
 529
 530             result += line + "\n"
 531
 532         return result
 533
 534     def prepareSubmitTemplate(self):
 535         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 536         template = ""
 537         inFilesSection = False
 538         for line in p4_read_pipe_lines("change -o"):
 539             if line.endswith("\r\n"):
 540                 line = line[:-2] + "\n"
 541             if inFilesSection:
 542                 if line.startswith("\t"):
 543                     # path starts and ends with a tab
 544                     path = line[1:]
 545                     lastTab = path.rfind("\t")
 546                     if lastTab != -1:
 547                         path = path[:lastTab]
 548                         if not path.startswith(self.depotPath):
 549                             continue
 550                 else:
 551                     inFilesSection = False
 552             else:
 553                 if line.startswith("Files:"):
 554                     inFilesSection = True
 555
 556             template += line
 557
 558         return template
 559
 560     def applyCommit(self, id):
 561         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 562         diffOpts = ("", "-M")[self.detectRename]
 563         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 564         filesToAdd = set()
 565         filesToDelete = set()
 566         editedFiles = set()
 567         filesToChangeExecBit = {}
 568         for line in diff:
 569             diff = parseDiffTreeEntry(line)
 570             modifier = diff['status']
 571             path = diff['src']
 572             if modifier == "M":
 573                 p4_system("edit \"%s\"" % path)
 574                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 575                     filesToChangeExecBit[path] = diff['dst_mode']
 576                 editedFiles.add(path)
 577             elif modifier == "A":
 578                 filesToAdd.add(path)
 579                 filesToChangeExecBit[path] = diff['dst_mode']
 580                 if path in filesToDelete:
 581                     filesToDelete.remove(path)
 582             elif modifier == "D":
 583                 filesToDelete.add(path)
 584                 if path in filesToAdd:
 585                     filesToAdd.remove(path)
 586             elif modifier == "R":
 587                 src, dest = diff['src'], diff['dst']
 588                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 589                 p4_system("edit \"%s\"" % (dest))
 590                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 591                     filesToChangeExecBit[dest] = diff['dst_mode']
 592                 os.unlink(dest)
 593                 editedFiles.add(dest)
 594                 filesToDelete.add(src)
 595             else:
 596                 die("unknown modifier %s for %s" % (modifier, path))
 597
 598         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 599         patchcmd = diffcmd + " | git apply "
 600         tryPatchCmd = patchcmd + "--check -"
 601         applyPatchCmd = patchcmd + "--check --apply -"
 602
 603         if os.system(tryPatchCmd) != 0:
 604             print "Unfortunately applying the change failed!"
 605             print "What do you want to do?"
 606             response = "x"
 607             while response != "s" and response != "a" and response != "w":
 608                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 609                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 610             if response == "s":
 611                 print "Skipping! Good luck with the next patches..."
 612                 for f in editedFiles:
 613                     p4_system("revert \"%s\"" % f);
 614                 for f in filesToAdd:
 615                     system("rm %s" %f)
 616                 return
 617             elif response == "a":
 618                 os.system(applyPatchCmd)
 619                 if len(filesToAdd) > 0:
 620                     print "You may also want to call p4 add on the following files:"
 621                     print " ".join(filesToAdd)
 622                 if len(filesToDelete):
 623                     print "The following files should be scheduled for deletion with p4 delete:"
 624                     print " ".join(filesToDelete)
 625                 die("Please resolve and submit the conflict manually and "
 626                     + "continue afterwards with git-p4 submit --continue")
 627             elif response == "w":
 628                 system(diffcmd + " > patch.txt")
 629                 print "Patch saved to patch.txt in %s !" % self.clientPath
 630                 die("Please resolve and submit the conflict manually and "
 631                     "continue afterwards with git-p4 submit --continue")
 632
 633         system(applyPatchCmd)
 634
 635         for f in filesToAdd:
 636             p4_system("add \"%s\"" % f)
 637         for f in filesToDelete:
 638             p4_system("revert \"%s\"" % f)
 639             p4_system("delete \"%s\"" % f)
 640
 641         # Set/clear executable bits
 642         for f in filesToChangeExecBit.keys():
 643             mode = filesToChangeExecBit[f]
 644             setP4ExecBit(f, mode)
 645
 646         logMessage = extractLogMessageFromGitCommit(id)
 647         logMessage = logMessage.strip()
 648
 649         template = self.prepareSubmitTemplate()
 650
 651         if self.interactive:
 652             submitTemplate = self.prepareLogMessage(template, logMessage)
 653             if os.environ.has_key("P4DIFF"):
 654                 del(os.environ["P4DIFF"])
 655             diff = read_pipe("p4 diff -du ...")
 656
 657             newdiff = ""
 658             for newFile in filesToAdd:
 659                 newdiff += "==== new file ====\n"
 660                 newdiff += "--- /dev/null\n"
 661                 newdiff += "+++ %s\n" % newFile
 662                 f = open(newFile, "r")
 663                 for line in f.readlines():
 664                     newdiff += "+" + line
 665                 f.close()
 666
 667             separatorLine = "######## everything below this line is just the diff #######\n"
 668
 669             [handle, fileName] = tempfile.mkstemp()
 670             tmpFile = os.fdopen(handle, "w+")
 671             if self.isWindows:
 672                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 673                 separatorLine = separatorLine.replace("\n", "\r\n")
 674                 newdiff = newdiff.replace("\n", "\r\n")
 675             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 676             tmpFile.close()
 677             defaultEditor = "vi"
 678             if platform.system() == "Windows":
 679                 defaultEditor = "notepad"
 680             if os.environ.has_key("P4EDITOR"):
 681                 editor = os.environ.get("P4EDITOR")
 682             else:
 683                 editor = os.environ.get("EDITOR", defaultEditor);
 684             system(editor + " " + fileName)
 685             tmpFile = open(fileName, "rb")
 686             message = tmpFile.read()
 687             tmpFile.close()
 688             os.remove(fileName)
 689             submitTemplate = message[:message.index(separatorLine)]
 690             if self.isWindows:
 691                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 692
 693             write_pipe("p4 submit -i", submitTemplate)
 694         else:
 695             fileName = "submit.txt"
 696             file = open(fileName, "w+")
 697             file.write(self.prepareLogMessage(template, logMessage))
 698             file.close()
 699             print ("Perforce submit template written as %s. "
 700                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 701                    % (fileName, fileName))
 702
 703     def run(self, args):
 704         if len(args) == 0:
 705             self.master = currentGitBranch()
 706             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 707                 die("Detecting current git branch failed!")
 708         elif len(args) == 1:
 709             self.master = args[0]
 710         else:
 711             return False
 712
 713         allowSubmit = gitConfig("git-p4.allowSubmit")
 714         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 715             die("%s is not in git-p4.allowSubmit" % self.master)
 716
 717         [upstream, settings] = findUpstreamBranchPoint()
 718         self.depotPath = settings['depot-paths'][0]
 719         if len(self.origin) == 0:
 720             self.origin = upstream
 721
 722         if self.verbose:
 723             print "Origin branch is " + self.origin
 724
 725         if len(self.depotPath) == 0:
 726             print "Internal error: cannot locate perforce depot path from existing branches"
 727             sys.exit(128)
 728
 729         self.clientPath = p4Where(self.depotPath)
 730
 731         if len(self.clientPath) == 0:
 732             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 733             sys.exit(128)
 734
 735         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 736         self.oldWorkingDirectory = os.getcwd()
 737
 738         os.chdir(self.clientPath)
 739         print "Syncronizing p4 checkout..."
 740         p4_system("sync ...")
 741
 742         self.check()
 743
 744         commits = []
 745         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 746             commits.append(line.strip())
 747         commits.reverse()
 748
 749         while len(commits) > 0:
 750             commit = commits[0]
 751             commits = commits[1:]
 752             self.applyCommit(commit)
 753             if not self.interactive:
 754                 break
 755
 756         if len(commits) == 0:
 757             print "All changes applied!"
 758             os.chdir(self.oldWorkingDirectory)
 759
 760             sync = P4Sync()
 761             sync.run([])
 762
 763             rebase = P4Rebase()
 764             rebase.rebase()
 765
 766         return True
 767
 768 class P4Sync(Command):
 769     def __init__(self):
 770         Command.__init__(self)
 771         self.options = [
 772                 optparse.make_option("--branch", dest="branch"),
 773                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 774                 optparse.make_option("--changesfile", dest="changesFile"),
 775                 optparse.make_option("--silent", dest="silent", action="store_true"),
 776                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 777                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 778                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 779                                      help="Import into refs/heads/ , not refs/remotes"),
 780                 optparse.make_option("--max-changes", dest="maxChanges"),
 781                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 782                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 783                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 784                                      help="Only sync files that are included in the Perforce Client Spec")
 785         ]
 786         self.description = """Imports from Perforce into a git repository.\n
 787     example:
 788     //depot/my/project/ -- to import the current head
 789     //depot/my/project/@all -- to import everything
 790     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 791
 792     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 793
 794         self.usage += " //depot/path[@revRange]"
 795         self.silent = False
 796         self.createdBranches = Set()
 797         self.committedChanges = Set()
 798         self.branch = ""
 799         self.detectBranches = False
 800         self.detectLabels = False
 801         self.changesFile = ""
 802         self.syncWithOrigin = True
 803         self.verbose = False
 804         self.importIntoRemotes = True
 805         self.maxChanges = ""
 806         self.isWindows = (platform.system() == "Windows")
 807         self.keepRepoPath = False
 808         self.depotPaths = None
 809         self.p4BranchesInGit = []
 810         self.cloneExclude = []
 811         self.useClientSpec = False
 812         self.clientSpecDirs = []
 813
 814         if gitConfig("git-p4.syncFromOrigin") == "false":
 815             self.syncWithOrigin = False
 816
 817     def extractFilesFromCommit(self, commit):
 818         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 819                              for path in self.cloneExclude]
 820         files = []
 821         fnum = 0
 822         while commit.has_key("depotFile%s" % fnum):
 823             path =  commit["depotFile%s" % fnum]
 824
 825             if [p for p in self.cloneExclude
 826                 if path.startswith (p)]:
 827                 found = False
 828             else:
 829                 found = [p for p in self.depotPaths
 830                          if path.startswith (p)]
 831             if not found:
 832                 fnum = fnum + 1
 833                 continue
 834
 835             file = {}
 836             file["path"] = path
 837             file["rev"] = commit["rev%s" % fnum]
 838             file["action"] = commit["action%s" % fnum]
 839             file["type"] = commit["type%s" % fnum]
 840             files.append(file)
 841             fnum = fnum + 1
 842         return files
 843
 844     def stripRepoPath(self, path, prefixes):
 845         if self.keepRepoPath:
 846             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 847
 848         for p in prefixes:
 849             if path.startswith(p):
 850                 path = path[len(p):]
 851
 852         return path
 853
 854     def splitFilesIntoBranches(self, commit):
 855         branches = {}
 856         fnum = 0
 857         while commit.has_key("depotFile%s" % fnum):
 858             path =  commit["depotFile%s" % fnum]
 859             found = [p for p in self.depotPaths
 860                      if path.startswith (p)]
 861             if not found:
 862                 fnum = fnum + 1
 863                 continue
 864
 865             file = {}
 866             file["path"] = path
 867             file["rev"] = commit["rev%s" % fnum]
 868             file["action"] = commit["action%s" % fnum]
 869             file["type"] = commit["type%s" % fnum]
 870             fnum = fnum + 1
 871
 872             relPath = self.stripRepoPath(path, self.depotPaths)
 873
 874             for branch in self.knownBranches.keys():
 875
 876                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 877                 if relPath.startswith(branch + "/"):
 878                     if branch not in branches:
 879                         branches[branch] = []
 880                     branches[branch].append(file)
 881                     break
 882
 883         return branches
 884
 885     ## Should move this out, doesn't use SELF.
 886     def readP4Files(self, files):
 887         filesForCommit = []
 888         filesToRead = []
 889
 890         for f in files:
 891             includeFile = True
 892             for val in self.clientSpecDirs:
 893                 if f['path'].startswith(val[0]):
 894                     if val[1] <= 0:
 895                         includeFile = False
 896                     break
 897
 898             if includeFile:
 899                 filesForCommit.append(f)
 900                 if f['action'] != 'delete':
 901                     filesToRead.append(f)
 902
 903         filedata = []
 904         if len(filesToRead) > 0:
 905             filedata = p4CmdList('-x - print',
 906                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 907                                                   for f in filesToRead]),
 908                                  stdin_mode='w+')
 909
 910             if "p4ExitCode" in filedata[0]:
 911                 die("Problems executing p4. Error: [%d]."
 912                     % (filedata[0]['p4ExitCode']));
 913
 914         j = 0;
 915         contents = {}
 916         while j < len(filedata):
 917             stat = filedata[j]
 918             j += 1
 919             text = [];
 920             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 921                 text.append(filedata[j]['data'])
 922                 j += 1
 923             text = ''.join(text)
 924
 925             if not stat.has_key('depotFile'):
 926                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 927                 continue
 928
 929             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 930                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 931             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 932                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 933
 934             contents[stat['depotFile']] = text
 935
 936         for f in filesForCommit:
 937             path = f['path']
 938             if contents.has_key(path):
 939                 f['data'] = contents[path]
 940
 941         return filesForCommit
 942
 943     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 944         epoch = details["time"]
 945         author = details["user"]
 946
 947         if self.verbose:
 948             print "commit into %s" % branch
 949
 950         # start with reading files; if that fails, we should not
 951         # create a commit.
 952         new_files = []
 953         for f in files:
 954             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 955                 new_files.append (f)
 956             else:
 957                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 958         files = self.readP4Files(new_files)
 959
 960         self.gitStream.write("commit %s\n" % branch)
 961 #        gitStream.write("mark :%s\n" % details["change"])
 962         self.committedChanges.add(int(details["change"]))
 963         committer = ""
 964         if author not in self.users:
 965             self.getUserMapFromPerforceServer()
 966         if author in self.users:
 967             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 968         else:
 969             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 970
 971         self.gitStream.write("committer %s\n" % committer)
 972
 973         self.gitStream.write("data <<EOT\n")
 974         self.gitStream.write(details["desc"])
 975         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 976                              % (','.join (branchPrefixes), details["change"]))
 977         if len(details['options']) > 0:
 978             self.gitStream.write(": options = %s" % details['options'])
 979         self.gitStream.write("]\nEOT\n\n")
 980
 981         if len(parent) > 0:
 982             if self.verbose:
 983                 print "parent %s" % parent
 984             self.gitStream.write("from %s\n" % parent)
 985
 986         for file in files:
 987             if file["type"] == "apple":
 988                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 989                 continue
 990
 991             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 992             if file["action"] == "delete":
 993                 self.gitStream.write("D %s\n" % relPath)
 994             else:
 995                 data = file['data']
 996
 997                 mode = "644"
 998                 if isP4Exec(file["type"]):
 999                     mode = "755"
1000                 elif file["type"] == "symlink":
1001                     mode = "120000"
1002                     # p4 print on a symlink contains "target\n", so strip it off
1003                     data = data[:-1]
1004
1005                 if self.isWindows and file["type"].endswith("text"):
1006                     data = data.replace("\r\n", "\n")
1007
1008                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1009                 self.gitStream.write("data %s\n" % len(data))
1010                 self.gitStream.write(data)
1011                 self.gitStream.write("\n")
1012
1013         self.gitStream.write("\n")
1014
1015         change = int(details["change"])
1016
1017         if self.labels.has_key(change):
1018             label = self.labels[change]
1019             labelDetails = label[0]
1020             labelRevisions = label[1]
1021             if self.verbose:
1022                 print "Change %s is labelled %s" % (change, labelDetails)
1023
1024             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1025                                                     for p in branchPrefixes]))
1026
1027             if len(files) == len(labelRevisions):
1028
1029                 cleanedFiles = {}
1030                 for info in files:
1031                     if info["action"] == "delete":
1032                         continue
1033                     cleanedFiles[info["depotFile"]] = info["rev"]
1034
1035                 if cleanedFiles == labelRevisions:
1036                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1037                     self.gitStream.write("from %s\n" % branch)
1038
1039                     owner = labelDetails["Owner"]
1040                     tagger = ""
1041                     if author in self.users:
1042                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1043                     else:
1044                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1045                     self.gitStream.write("tagger %s\n" % tagger)
1046                     self.gitStream.write("data <<EOT\n")
1047                     self.gitStream.write(labelDetails["Description"])
1048                     self.gitStream.write("EOT\n\n")
1049
1050                 else:
1051                     if not self.silent:
1052                         print ("Tag %s does not match with change %s: files do not match."
1053                                % (labelDetails["label"], change))
1054
1055             else:
1056                 if not self.silent:
1057                     print ("Tag %s does not match with change %s: file count is different."
1058                            % (labelDetails["label"], change))
1059
1060     def getUserCacheFilename(self):
1061         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1062         return home + "/.gitp4-usercache.txt"
1063
1064     def getUserMapFromPerforceServer(self):
1065         if self.userMapFromPerforceServer:
1066             return
1067         self.users = {}
1068
1069         for output in p4CmdList("users"):
1070             if not output.has_key("User"):
1071                 continue
1072             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1073
1074
1075         s = ''
1076         for (key, val) in self.users.items():
1077             s += "%s\t%s\n" % (key, val)
1078
1079         open(self.getUserCacheFilename(), "wb").write(s)
1080         self.userMapFromPerforceServer = True
1081
1082     def loadUserMapFromCache(self):
1083         self.users = {}
1084         self.userMapFromPerforceServer = False
1085         try:
1086             cache = open(self.getUserCacheFilename(), "rb")
1087             lines = cache.readlines()
1088             cache.close()
1089             for line in lines:
1090                 entry = line.strip().split("\t")
1091                 self.users[entry[0]] = entry[1]
1092         except IOError:
1093             self.getUserMapFromPerforceServer()
1094
1095     def getLabels(self):
1096         self.labels = {}
1097
1098         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1099         if len(l) > 0 and not self.silent:
1100             print "Finding files belonging to labels in %s" % `self.depotPaths`
1101
1102         for output in l:
1103             label = output["label"]
1104             revisions = {}
1105             newestChange = 0
1106             if self.verbose:
1107                 print "Querying files for label %s" % label
1108             for file in p4CmdList("files "
1109                                   +  ' '.join (["%s...@%s" % (p, label)
1110                                                 for p in self.depotPaths])):
1111                 revisions[file["depotFile"]] = file["rev"]
1112                 change = int(file["change"])
1113                 if change > newestChange:
1114                     newestChange = change
1115
1116             self.labels[newestChange] = [output, revisions]
1117
1118         if self.verbose:
1119             print "Label changes: %s" % self.labels.keys()
1120
1121     def guessProjectName(self):
1122         for p in self.depotPaths:
1123             if p.endswith("/"):
1124                 p = p[:-1]
1125             p = p[p.strip().rfind("/") + 1:]
1126             if not p.endswith("/"):
1127                p += "/"
1128             return p
1129
1130     def getBranchMapping(self):
1131         lostAndFoundBranches = set()
1132
1133         for info in p4CmdList("branches"):
1134             details = p4Cmd("branch -o %s" % info["branch"])
1135             viewIdx = 0
1136             while details.has_key("View%s" % viewIdx):
1137                 paths = details["View%s" % viewIdx].split(" ")
1138                 viewIdx = viewIdx + 1
1139                 # require standard //depot/foo/... //depot/bar/... mapping
1140                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1141                     continue
1142                 source = paths[0]
1143                 destination = paths[1]
1144                 ## HACK
1145                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1146                     source = source[len(self.depotPaths[0]):-4]
1147                     destination = destination[len(self.depotPaths[0]):-4]
1148
1149                     if destination in self.knownBranches:
1150                         if not self.silent:
1151                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1152                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1153                         continue
1154
1155                     self.knownBranches[destination] = source
1156
1157                     lostAndFoundBranches.discard(destination)
1158
1159                     if source not in self.knownBranches:
1160                         lostAndFoundBranches.add(source)
1161
1162
1163         for branch in lostAndFoundBranches:
1164             self.knownBranches[branch] = branch
1165
1166     def getBranchMappingFromGitBranches(self):
1167         branches = p4BranchesInGit(self.importIntoRemotes)
1168         for branch in branches.keys():
1169             if branch == "master":
1170                 branch = "main"
1171             else:
1172                 branch = branch[len(self.projectName):]
1173             self.knownBranches[branch] = branch
1174
1175     def listExistingP4GitBranches(self):
1176         # branches holds mapping from name to commit
1177         branches = p4BranchesInGit(self.importIntoRemotes)
1178         self.p4BranchesInGit = branches.keys()
1179         for branch in branches.keys():
1180             self.initialParents[self.refPrefix + branch] = branches[branch]
1181
1182     def updateOptionDict(self, d):
1183         option_keys = {}
1184         if self.keepRepoPath:
1185             option_keys['keepRepoPath'] = 1
1186
1187         d["options"] = ' '.join(sorted(option_keys.keys()))
1188
1189     def readOptions(self, d):
1190         self.keepRepoPath = (d.has_key('options')
1191                              and ('keepRepoPath' in d['options']))
1192
1193     def gitRefForBranch(self, branch):
1194         if branch == "main":
1195             return self.refPrefix + "master"
1196
1197         if len(branch) <= 0:
1198             return branch
1199
1200         return self.refPrefix + self.projectName + branch
1201
1202     def gitCommitByP4Change(self, ref, change):
1203         if self.verbose:
1204             print "looking in ref " + ref + " for change %s using bisect..." % change
1205
1206         earliestCommit = ""
1207         latestCommit = parseRevision(ref)
1208
1209         while True:
1210             if self.verbose:
1211                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1212             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1213             if len(next) == 0:
1214                 if self.verbose:
1215                     print "argh"
1216                 return ""
1217             log = extractLogMessageFromGitCommit(next)
1218             settings = extractSettingsGitLog(log)
1219             currentChange = int(settings['change'])
1220             if self.verbose:
1221                 print "current change %s" % currentChange
1222
1223             if currentChange == change:
1224                 if self.verbose:
1225                     print "found %s" % next
1226                 return next
1227
1228             if currentChange < change:
1229                 earliestCommit = "^%s" % next
1230             else:
1231                 latestCommit = "%s" % next
1232
1233         return ""
1234
1235     def importNewBranch(self, branch, maxChange):
1236         # make fast-import flush all changes to disk and update the refs using the checkpoint
1237         # command so that we can try to find the branch parent in the git history
1238         self.gitStream.write("checkpoint\n\n");
1239         self.gitStream.flush();
1240         branchPrefix = self.depotPaths[0] + branch + "/"
1241         range = "@1,%s" % maxChange
1242         #print "prefix" + branchPrefix
1243         changes = p4ChangesForPaths([branchPrefix], range)
1244         if len(changes) <= 0:
1245             return False
1246         firstChange = changes[0]
1247         #print "first change in branch: %s" % firstChange
1248         sourceBranch = self.knownBranches[branch]
1249         sourceDepotPath = self.depotPaths[0] + sourceBranch
1250         sourceRef = self.gitRefForBranch(sourceBranch)
1251         #print "source " + sourceBranch
1252
1253         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1254         #print "branch parent: %s" % branchParentChange
1255         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1256         if len(gitParent) > 0:
1257             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1258             #print "parent git commit: %s" % gitParent
1259
1260         self.importChanges(changes)
1261         return True
1262
1263     def importChanges(self, changes):
1264         cnt = 1
1265         for change in changes:
1266             description = p4Cmd("describe %s" % change)
1267             self.updateOptionDict(description)
1268
1269             if not self.silent:
1270                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1271                 sys.stdout.flush()
1272             cnt = cnt + 1
1273
1274             try:
1275                 if self.detectBranches:
1276                     branches = self.splitFilesIntoBranches(description)
1277                     for branch in branches.keys():
1278                         ## HACK  --hwn
1279                         branchPrefix = self.depotPaths[0] + branch + "/"
1280
1281                         parent = ""
1282
1283                         filesForCommit = branches[branch]
1284
1285                         if self.verbose:
1286                             print "branch is %s" % branch
1287
1288                         self.updatedBranches.add(branch)
1289
1290                         if branch not in self.createdBranches:
1291                             self.createdBranches.add(branch)
1292                             parent = self.knownBranches[branch]
1293                             if parent == branch:
1294                                 parent = ""
1295                             else:
1296                                 fullBranch = self.projectName + branch
1297                                 if fullBranch not in self.p4BranchesInGit:
1298                                     if not self.silent:
1299                                         print("\n    Importing new branch %s" % fullBranch);
1300                                     if self.importNewBranch(branch, change - 1):
1301                                         parent = ""
1302                                         self.p4BranchesInGit.append(fullBranch)
1303                                     if not self.silent:
1304                                         print("\n    Resuming with change %s" % change);
1305
1306                                 if self.verbose:
1307                                     print "parent determined through known branches: %s" % parent
1308
1309                         branch = self.gitRefForBranch(branch)
1310                         parent = self.gitRefForBranch(parent)
1311
1312                         if self.verbose:
1313                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1314
1315                         if len(parent) == 0 and branch in self.initialParents:
1316                             parent = self.initialParents[branch]
1317                             del self.initialParents[branch]
1318
1319                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1320                 else:
1321                     files = self.extractFilesFromCommit(description)
1322                     self.commit(description, files, self.branch, self.depotPaths,
1323                                 self.initialParent)
1324                     self.initialParent = ""
1325             except IOError:
1326                 print self.gitError.read()
1327                 sys.exit(1)
1328
1329     def importHeadRevision(self, revision):
1330         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1331
1332         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1333         details["desc"] = ("Initial import of %s from the state at revision %s"
1334                            % (' '.join(self.depotPaths), revision))
1335         details["change"] = revision
1336         newestRevision = 0
1337
1338         fileCnt = 0
1339         for info in p4CmdList("files "
1340                               +  ' '.join(["%s...%s"
1341                                            % (p, revision)
1342                                            for p in self.depotPaths])):
1343
1344             if info['code'] == 'error':
1345                 sys.stderr.write("p4 returned an error: %s\n"
1346                                  % info['data'])
1347                 sys.exit(1)
1348
1349
1350             change = int(info["change"])
1351             if change > newestRevision:
1352                 newestRevision = change
1353
1354             if info["action"] == "delete":
1355                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1356                 #fileCnt = fileCnt + 1
1357                 continue
1358
1359             for prop in ["depotFile", "rev", "action", "type" ]:
1360                 details["%s%s" % (prop, fileCnt)] = info[prop]
1361
1362             fileCnt = fileCnt + 1
1363
1364         details["change"] = newestRevision
1365         self.updateOptionDict(details)
1366         try:
1367             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1368         except IOError:
1369             print "IO error with git fast-import. Is your git version recent enough?"
1370             print self.gitError.read()
1371
1372
1373     def getClientSpec(self):
1374         specList = p4CmdList( "client -o" )
1375         temp = {}
1376         for entry in specList:
1377             for k,v in entry.iteritems():
1378                 if k.startswith("View"):
1379                     if v.startswith('"'):
1380                         start = 1
1381                     else:
1382                         start = 0
1383                     index = v.find("...")
1384                     v = v[start:index]
1385                     if v.startswith("-"):
1386                         v = v[1:]
1387                         temp[v] = -len(v)
1388                     else:
1389                         temp[v] = len(v)
1390         self.clientSpecDirs = temp.items()
1391         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1392
1393     def run(self, args):
1394         self.depotPaths = []
1395         self.changeRange = ""
1396         self.initialParent = ""
1397         self.previousDepotPaths = []
1398
1399         # map from branch depot path to parent branch
1400         self.knownBranches = {}
1401         self.initialParents = {}
1402         self.hasOrigin = originP4BranchesExist()
1403         if not self.syncWithOrigin:
1404             self.hasOrigin = False
1405
1406         if self.importIntoRemotes:
1407             self.refPrefix = "refs/remotes/p4/"
1408         else:
1409             self.refPrefix = "refs/heads/p4/"
1410
1411         if self.syncWithOrigin and self.hasOrigin:
1412             if not self.silent:
1413                 print "Syncing with origin first by calling git fetch origin"
1414             system("git fetch origin")
1415
1416         if len(self.branch) == 0:
1417             self.branch = self.refPrefix + "master"
1418             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1419                 system("git update-ref %s refs/heads/p4" % self.branch)
1420                 system("git branch -D p4");
1421             # create it /after/ importing, when master exists
1422             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1423                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1424
1425         if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1426             self.getClientSpec()
1427
1428         # TODO: should always look at previous commits,
1429         # merge with previous imports, if possible.
1430         if args == []:
1431             if self.hasOrigin:
1432                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1433             self.listExistingP4GitBranches()
1434
1435             if len(self.p4BranchesInGit) > 1:
1436                 if not self.silent:
1437                     print "Importing from/into multiple branches"
1438                 self.detectBranches = True
1439
1440             if self.verbose:
1441                 print "branches: %s" % self.p4BranchesInGit
1442
1443             p4Change = 0
1444             for branch in self.p4BranchesInGit:
1445                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1446
1447                 settings = extractSettingsGitLog(logMsg)
1448
1449                 self.readOptions(settings)
1450                 if (settings.has_key('depot-paths')
1451                     and settings.has_key ('change')):
1452                     change = int(settings['change']) + 1
1453                     p4Change = max(p4Change, change)
1454
1455                     depotPaths = sorted(settings['depot-paths'])
1456                     if self.previousDepotPaths == []:
1457                         self.previousDepotPaths = depotPaths
1458                     else:
1459                         paths = []
1460                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1461                             for i in range(0, min(len(cur), len(prev))):
1462                                 if cur[i] <> prev[i]:
1463                                     i = i - 1
1464                                     break
1465
1466                             paths.append (cur[:i + 1])
1467
1468                         self.previousDepotPaths = paths
1469
1470             if p4Change > 0:
1471                 self.depotPaths = sorted(self.previousDepotPaths)
1472                 self.changeRange = "@%s,#head" % p4Change
1473                 if not self.detectBranches:
1474                     self.initialParent = parseRevision(self.branch)
1475                 if not self.silent and not self.detectBranches:
1476                     print "Performing incremental import into %s git branch" % self.branch
1477
1478         if not self.branch.startswith("refs/"):
1479             self.branch = "refs/heads/" + self.branch
1480
1481         if len(args) == 0 and self.depotPaths:
1482             if not self.silent:
1483                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1484         else:
1485             if self.depotPaths and self.depotPaths != args:
1486                 print ("previous import used depot path %s and now %s was specified. "
1487                        "This doesn't work!" % (' '.join (self.depotPaths),
1488                                                ' '.join (args)))
1489                 sys.exit(1)
1490
1491             self.depotPaths = sorted(args)
1492
1493         revision = ""
1494         self.users = {}
1495
1496         newPaths = []
1497         for p in self.depotPaths:
1498             if p.find("@") != -1:
1499                 atIdx = p.index("@")
1500                 self.changeRange = p[atIdx:]
1501                 if self.changeRange == "@all":
1502                     self.changeRange = ""
1503                 elif ',' not in self.changeRange:
1504                     revision = self.changeRange
1505                     self.changeRange = ""
1506                 p = p[:atIdx]
1507             elif p.find("#") != -1:
1508                 hashIdx = p.index("#")
1509                 revision = p[hashIdx:]
1510                 p = p[:hashIdx]
1511             elif self.previousDepotPaths == []:
1512                 revision = "#head"
1513
1514             p = re.sub ("\.\.\.$", "", p)
1515             if not p.endswith("/"):
1516                 p += "/"
1517
1518             newPaths.append(p)
1519
1520         self.depotPaths = newPaths
1521
1522
1523         self.loadUserMapFromCache()
1524         self.labels = {}
1525         if self.detectLabels:
1526             self.getLabels();
1527
1528         if self.detectBranches:
1529             ## FIXME - what's a P4 projectName ?
1530             self.projectName = self.guessProjectName()
1531
1532             if self.hasOrigin:
1533                 self.getBranchMappingFromGitBranches()
1534             else:
1535                 self.getBranchMapping()
1536             if self.verbose:
1537                 print "p4-git branches: %s" % self.p4BranchesInGit
1538                 print "initial parents: %s" % self.initialParents
1539             for b in self.p4BranchesInGit:
1540                 if b != "master":
1541
1542                     ## FIXME
1543                     b = b[len(self.projectName):]
1544                 self.createdBranches.add(b)
1545
1546         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1547
1548         importProcess = subprocess.Popen(["git", "fast-import"],
1549                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1550                                          stderr=subprocess.PIPE);
1551         self.gitOutput = importProcess.stdout
1552         self.gitStream = importProcess.stdin
1553         self.gitError = importProcess.stderr
1554
1555         if revision:
1556             self.importHeadRevision(revision)
1557         else:
1558             changes = []
1559
1560             if len(self.changesFile) > 0:
1561                 output = open(self.changesFile).readlines()
1562                 changeSet = Set()
1563                 for line in output:
1564                     changeSet.add(int(line))
1565
1566                 for change in changeSet:
1567                     changes.append(change)
1568
1569                 changes.sort()
1570             else:
1571                 if self.verbose:
1572                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1573                                                               self.changeRange)
1574                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1575
1576                 if len(self.maxChanges) > 0:
1577                     changes = changes[:min(int(self.maxChanges), len(changes))]
1578
1579             if len(changes) == 0:
1580                 if not self.silent:
1581                     print "No changes to import!"
1582                 return True
1583
1584             if not self.silent and not self.detectBranches:
1585                 print "Import destination: %s" % self.branch
1586
1587             self.updatedBranches = set()
1588
1589             self.importChanges(changes)
1590
1591             if not self.silent:
1592                 print ""
1593                 if len(self.updatedBranches) > 0:
1594                     sys.stdout.write("Updated branches: ")
1595                     for b in self.updatedBranches:
1596                         sys.stdout.write("%s " % b)
1597                     sys.stdout.write("\n")
1598
1599         self.gitStream.close()
1600         if importProcess.wait() != 0:
1601             die("fast-import failed: %s" % self.gitError.read())
1602         self.gitOutput.close()
1603         self.gitError.close()
1604
1605         return True
1606
1607 class P4Rebase(Command):
1608     def __init__(self):
1609         Command.__init__(self)
1610         self.options = [ ]
1611         self.description = ("Fetches the latest revision from perforce and "
1612                             + "rebases the current work (branch) against it")
1613         self.verbose = False
1614
1615     def run(self, args):
1616         sync = P4Sync()
1617         sync.run([])
1618
1619         return self.rebase()
1620
1621     def rebase(self):
1622         if os.system("git update-index --refresh") != 0:
1623             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1624         if len(read_pipe("git diff-index HEAD --")) > 0:
1625             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1626
1627         [upstream, settings] = findUpstreamBranchPoint()
1628         if len(upstream) == 0:
1629             die("Cannot find upstream branchpoint for rebase")
1630
1631         # the branchpoint may be p4/foo~3, so strip off the parent
1632         upstream = re.sub("~[0-9]+$", "", upstream)
1633
1634         print "Rebasing the current branch onto %s" % upstream
1635         oldHead = read_pipe("git rev-parse HEAD").strip()
1636         system("git rebase %s" % upstream)
1637         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1638         return True
1639
1640 class P4Clone(P4Sync):
1641     def __init__(self):
1642         P4Sync.__init__(self)
1643         self.description = "Creates a new git repository and imports from Perforce into it"
1644         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1645         self.options += [
1646             optparse.make_option("--destination", dest="cloneDestination",
1647                                  action='store', default=None,
1648                                  help="where to leave result of the clone"),
1649             optparse.make_option("-/", dest="cloneExclude",
1650                                  action="append", type="string",
1651                                  help="exclude depot path")
1652         ]
1653         self.cloneDestination = None
1654         self.needsGit = False
1655
1656     # This is required for the "append" cloneExclude action
1657     def ensure_value(self, attr, value):
1658         if not hasattr(self, attr) or getattr(self, attr) is None:
1659             setattr(self, attr, value)
1660         return getattr(self, attr)
1661
1662     def defaultDestination(self, args):
1663         ## TODO: use common prefix of args?
1664         depotPath = args[0]
1665         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1666         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1667         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1668         depotDir = re.sub(r"/$", "", depotDir)
1669         return os.path.split(depotDir)[1]
1670
1671     def run(self, args):
1672         if len(args) < 1:
1673             return False
1674
1675         if self.keepRepoPath and not self.cloneDestination:
1676             sys.stderr.write("Must specify destination for --keep-path\n")
1677             sys.exit(1)
1678
1679         depotPaths = args
1680
1681         if not self.cloneDestination and len(depotPaths) > 1:
1682             self.cloneDestination = depotPaths[-1]
1683             depotPaths = depotPaths[:-1]
1684
1685         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1686         for p in depotPaths:
1687             if not p.startswith("//"):
1688                 return False
1689
1690         if not self.cloneDestination:
1691             self.cloneDestination = self.defaultDestination(args)
1692
1693         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1694         if not os.path.exists(self.cloneDestination):
1695             os.makedirs(self.cloneDestination)
1696         os.chdir(self.cloneDestination)
1697         system("git init")
1698         self.gitdir = os.getcwd() + "/.git"
1699         if not P4Sync.run(self, depotPaths):
1700             return False
1701         if self.branch != "master":
1702             if gitBranchExists("refs/remotes/p4/master"):
1703                 system("git branch master refs/remotes/p4/master")
1704                 system("git checkout -f")
1705             else:
1706                 print "Could not detect main branch. No checkout/master branch created."
1707
1708         return True
1709
1710 class P4Branches(Command):
1711     def __init__(self):
1712         Command.__init__(self)
1713         self.options = [ ]
1714         self.description = ("Shows the git branches that hold imports and their "
1715                             + "corresponding perforce depot paths")
1716         self.verbose = False
1717
1718     def run(self, args):
1719         if originP4BranchesExist():
1720             createOrUpdateBranchesFromOrigin()
1721
1722         cmdline = "git rev-parse --symbolic "
1723         cmdline += " --remotes"
1724
1725         for line in read_pipe_lines(cmdline):
1726             line = line.strip()
1727
1728             if not line.startswith('p4/') or line == "p4/HEAD":
1729                 continue
1730             branch = line
1731
1732             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1733             settings = extractSettingsGitLog(log)
1734
1735             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1736         return True
1737
1738 class HelpFormatter(optparse.IndentedHelpFormatter):
1739     def __init__(self):
1740         optparse.IndentedHelpFormatter.__init__(self)
1741
1742     def format_description(self, description):
1743         if description:
1744             return description + "\n"
1745         else:
1746             return ""
1747
1748 def printUsage(commands):
1749     print "usage: %s <command> [options]" % sys.argv[0]
1750     print ""
1751     print "valid commands: %s" % ", ".join(commands)
1752     print ""
1753     print "Try %s <command> --help for command specific help." % sys.argv[0]
1754     print ""
1755
1756 commands = {
1757     "debug" : P4Debug,
1758     "submit" : P4Submit,
1759     "commit" : P4Submit,
1760     "sync" : P4Sync,
1761     "rebase" : P4Rebase,
1762     "clone" : P4Clone,
1763     "rollback" : P4RollBack,
1764     "branches" : P4Branches
1765 }
1766
1767
1768 def main():
1769     if len(sys.argv[1:]) == 0:
1770         printUsage(commands.keys())
1771         sys.exit(2)
1772
1773     cmd = ""
1774     cmdName = sys.argv[1]
1775     try:
1776         klass = commands[cmdName]
1777         cmd = klass()
1778     except KeyError:
1779         print "unknown command %s" % cmdName
1780         print ""
1781         printUsage(commands.keys())
1782         sys.exit(2)
1783
1784     options = cmd.options
1785     cmd.gitdir = os.environ.get("GIT_DIR", None)
1786
1787     args = sys.argv[2:]
1788
1789     if len(options) > 0:
1790         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1791
1792         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1793                                        options,
1794                                        description = cmd.description,
1795                                        formatter = HelpFormatter())
1796
1797         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1798     global verbose
1799     verbose = cmd.verbose
1800     if cmd.needsGit:
1801         if cmd.gitdir == None:
1802             cmd.gitdir = os.path.abspath(".git")
1803             if not isValidGitDir(cmd.gitdir):
1804                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1805                 if os.path.exists(cmd.gitdir):
1806                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1807                     if len(cdup) > 0:
1808                         os.chdir(cdup);
1809
1810         if not isValidGitDir(cmd.gitdir):
1811             if isValidGitDir(cmd.gitdir + "/.git"):
1812                 cmd.gitdir += "/.git"
1813             else:
1814                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1815
1816         os.environ["GIT_DIR"] = cmd.gitdir
1817
1818     if not cmd.run(args):
1819         parser.print_help()
1820
1821
1822 if __name__ == '__main__':
1823     main()