contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     if os.name == 'nt':
  56         os.environ['PWD']=dir
  57     os.chdir(dir)
  58
  59 def die(msg):
  60     if verbose:
  61         raise Exception(msg)
  62     else:
  63         sys.stderr.write(msg + "\n")
  64         sys.exit(1)
  65
  66 def write_pipe(c, str):
  67     if verbose:
  68         sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70     pipe = os.popen(c, 'w')
  71     val = pipe.write(str)
  72     if pipe.close():
  73         die('Command failed: %s' % c)
  74
  75     return val
  76
  77 def p4_write_pipe(c, str):
  78     real_cmd = p4_build_cmd(c)
  79     return write_pipe(real_cmd, str)
  80
  81 def read_pipe(c, ignore_error=False):
  82     if verbose:
  83         sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85     pipe = os.popen(c, 'rb')
  86     val = pipe.read()
  87     if pipe.close() and not ignore_error:
  88         die('Command failed: %s' % c)
  89
  90     return val
  91
  92 def p4_read_pipe(c, ignore_error=False):
  93     real_cmd = p4_build_cmd(c)
  94     return read_pipe(real_cmd, ignore_error)
  95
  96 def read_pipe_lines(c):
  97     if verbose:
  98         sys.stderr.write('Reading pipe: %s\n' % c)
  99     ## todo: check return status
 100     pipe = os.popen(c, 'rb')
 101     val = pipe.readlines()
 102     if pipe.close():
 103         die('Command failed: %s' % c)
 104
 105     return val
 106
 107 def p4_read_pipe_lines(c):
 108     """Specifically invoke p4 on the command supplied. """
 109     real_cmd = p4_build_cmd(c)
 110     return read_pipe_lines(real_cmd)
 111
 112 def system(cmd):
 113     if verbose:
 114         sys.stderr.write("executing %s\n" % cmd)
 115     if os.system(cmd) != 0:
 116         die("command failed: %s" % cmd)
 117
 118 def p4_system(cmd):
 119     """Specifically invoke p4 as the system command. """
 120     real_cmd = p4_build_cmd(cmd)
 121     return system(real_cmd)
 122
 123 def isP4Exec(kind):
 124     """Determine if a Perforce 'kind' should have execute permission
 125
 126     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128     a plus sign, it is also executable"""
 129     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131 def setP4ExecBit(file, mode):
 132     # Reopens an already open file and changes the execute bit to match
 133     # the execute bit setting in the passed in mode.
 134
 135     p4Type = "+x"
 136
 137     if not isModeExec(mode):
 138         p4Type = getP4OpenedType(file)
 139         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141         if p4Type[-1] == "+":
 142             p4Type = p4Type[0:-1]
 143
 144     p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146 def getP4OpenedType(file):
 147     # Returns the perforce file type for the given file.
 148
 149     result = p4_read_pipe("opened %s" % file)
 150     match = re.match(".*\((.+)\)\r?$", result)
 151     if match:
 152         return match.group(1)
 153     else:
 154         die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156 def diffTreePattern():
 157     # This is a simple generator for the diff tree regex pattern. This could be
 158     # a class variable if this and parseDiffTreeEntry were a part of a class.
 159     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160     while True:
 161         yield pattern
 162
 163 def parseDiffTreeEntry(entry):
 164     """Parses a single diff tree entry into its component elements.
 165
 166     See git-diff-tree(1) manpage for details about the format of the diff
 167     output. This method returns a dictionary with the following elements:
 168
 169     src_mode - The mode of the source file
 170     dst_mode - The mode of the destination file
 171     src_sha1 - The sha1 for the source file
 172     dst_sha1 - The sha1 fr the destination file
 173     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174     status_score - The score for the status (applicable for 'C' and 'R'
 175                    statuses). This is None if there is no score.
 176     src - The path for the source file.
 177     dst - The path for the destination file. This is only present for
 178           copy or renames. If it is not present, this is None.
 179
 180     If the pattern is not matched, None is returned."""
 181
 182     match = diffTreePattern().next().match(entry)
 183     if match:
 184         return {
 185             'src_mode': match.group(1),
 186             'dst_mode': match.group(2),
 187             'src_sha1': match.group(3),
 188             'dst_sha1': match.group(4),
 189             'status': match.group(5),
 190             'status_score': match.group(6),
 191             'src': match.group(7),
 192             'dst': match.group(10)
 193         }
 194     return None
 195
 196 def isModeExec(mode):
 197     # Returns True if the given git mode represents an executable file,
 198     # otherwise False.
 199     return mode[-3:] == "755"
 200
 201 def isModeExecChanged(src_mode, dst_mode):
 202     return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205     cmd = p4_build_cmd("-G %s" % (cmd))
 206     if verbose:
 207         sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209     # Use a temporary file to avoid deadlocks without
 210     # subprocess.communicate(), which would put another copy
 211     # of stdout into memory.
 212     stdin_file = None
 213     if stdin is not None:
 214         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215         stdin_file.write(stdin)
 216         stdin_file.flush()
 217         stdin_file.seek(0)
 218
 219     p4 = subprocess.Popen(cmd, shell=True,
 220                           stdin=stdin_file,
 221                           stdout=subprocess.PIPE)
 222
 223     result = []
 224     try:
 225         while True:
 226             entry = marshal.load(p4.stdout)
 227             result.append(entry)
 228     except EOFError:
 229         pass
 230     exitCode = p4.wait()
 231     if exitCode != 0:
 232         entry = {}
 233         entry["p4ExitCode"] = exitCode
 234         result.append(entry)
 235
 236     return result
 237
 238 def p4Cmd(cmd):
 239     list = p4CmdList(cmd)
 240     result = {}
 241     for entry in list:
 242         result.update(entry)
 243     return result;
 244
 245 def p4Where(depotPath):
 246     if not depotPath.endswith("/"):
 247         depotPath += "/"
 248     output = p4Cmd("where %s..." % depotPath)
 249     if output["code"] == "error":
 250         return ""
 251     clientPath = ""
 252     if "path" in output:
 253         clientPath = output.get("path")
 254     elif "data" in output:
 255         data = output.get("data")
 256         lastSpace = data.rfind(" ")
 257         clientPath = data[lastSpace + 1:]
 258
 259     if clientPath.endswith("..."):
 260         clientPath = clientPath[:-3]
 261     return clientPath
 262
 263 def currentGitBranch():
 264     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 265
 266 def isValidGitDir(path):
 267     if (os.path.exists(path + "/HEAD")
 268         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 269         return True;
 270     return False
 271
 272 def parseRevision(ref):
 273     return read_pipe("git rev-parse %s" % ref).strip()
 274
 275 def extractLogMessageFromGitCommit(commit):
 276     logMessage = ""
 277
 278     ## fixme: title is first line of commit, not 1st paragraph.
 279     foundTitle = False
 280     for log in read_pipe_lines("git cat-file commit %s" % commit):
 281        if not foundTitle:
 282            if len(log) == 1:
 283                foundTitle = True
 284            continue
 285
 286        logMessage += log
 287     return logMessage
 288
 289 def extractSettingsGitLog(log):
 290     values = {}
 291     for line in log.split("\n"):
 292         line = line.strip()
 293         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 294         if not m:
 295             continue
 296
 297         assignments = m.group(1).split (':')
 298         for a in assignments:
 299             vals = a.split ('=')
 300             key = vals[0].strip()
 301             val = ('='.join (vals[1:])).strip()
 302             if val.endswith ('\"') and val.startswith('"'):
 303                 val = val[1:-1]
 304
 305             values[key] = val
 306
 307     paths = values.get("depot-paths")
 308     if not paths:
 309         paths = values.get("depot-path")
 310     if paths:
 311         values['depot-paths'] = paths.split(',')
 312     return values
 313
 314 def gitBranchExists(branch):
 315     proc = subprocess.Popen(["git", "rev-parse", branch],
 316                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 317     return proc.wait() == 0;
 318
 319 _gitConfig = {}
 320 def gitConfig(key):
 321     if not _gitConfig.has_key(key):
 322         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 323     return _gitConfig[key]
 324
 325 def p4BranchesInGit(branchesAreInRemotes = True):
 326     branches = {}
 327
 328     cmdline = "git rev-parse --symbolic "
 329     if branchesAreInRemotes:
 330         cmdline += " --remotes"
 331     else:
 332         cmdline += " --branches"
 333
 334     for line in read_pipe_lines(cmdline):
 335         line = line.strip()
 336
 337         ## only import to p4/
 338         if not line.startswith('p4/') or line == "p4/HEAD":
 339             continue
 340         branch = line
 341
 342         # strip off p4
 343         branch = re.sub ("^p4/", "", line)
 344
 345         branches[branch] = parseRevision(line)
 346     return branches
 347
 348 def findUpstreamBranchPoint(head = "HEAD"):
 349     branches = p4BranchesInGit()
 350     # map from depot-path to branch name
 351     branchByDepotPath = {}
 352     for branch in branches.keys():
 353         tip = branches[branch]
 354         log = extractLogMessageFromGitCommit(tip)
 355         settings = extractSettingsGitLog(log)
 356         if settings.has_key("depot-paths"):
 357             paths = ",".join(settings["depot-paths"])
 358             branchByDepotPath[paths] = "remotes/p4/" + branch
 359
 360     settings = None
 361     parent = 0
 362     while parent < 65535:
 363         commit = head + "~%s" % parent
 364         log = extractLogMessageFromGitCommit(commit)
 365         settings = extractSettingsGitLog(log)
 366         if settings.has_key("depot-paths"):
 367             paths = ",".join(settings["depot-paths"])
 368             if branchByDepotPath.has_key(paths):
 369                 return [branchByDepotPath[paths], settings]
 370
 371         parent = parent + 1
 372
 373     return ["", settings]
 374
 375 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 376     if not silent:
 377         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 378                % localRefPrefix)
 379
 380     originPrefix = "origin/p4/"
 381
 382     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 383         line = line.strip()
 384         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 385             continue
 386
 387         headName = line[len(originPrefix):]
 388         remoteHead = localRefPrefix + headName
 389         originHead = line
 390
 391         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 392         if (not original.has_key('depot-paths')
 393             or not original.has_key('change')):
 394             continue
 395
 396         update = False
 397         if not gitBranchExists(remoteHead):
 398             if verbose:
 399                 print "creating %s" % remoteHead
 400             update = True
 401         else:
 402             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 403             if settings.has_key('change') > 0:
 404                 if settings['depot-paths'] == original['depot-paths']:
 405                     originP4Change = int(original['change'])
 406                     p4Change = int(settings['change'])
 407                     if originP4Change > p4Change:
 408                         print ("%s (%s) is newer than %s (%s). "
 409                                "Updating p4 branch from origin."
 410                                % (originHead, originP4Change,
 411                                   remoteHead, p4Change))
 412                         update = True
 413                 else:
 414                     print ("Ignoring: %s was imported from %s while "
 415                            "%s was imported from %s"
 416                            % (originHead, ','.join(original['depot-paths']),
 417                               remoteHead, ','.join(settings['depot-paths'])))
 418
 419         if update:
 420             system("git update-ref %s %s" % (remoteHead, originHead))
 421
 422 def originP4BranchesExist():
 423         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 424
 425 def p4ChangesForPaths(depotPaths, changeRange):
 426     assert depotPaths
 427     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 428                                                         for p in depotPaths]))
 429
 430     changes = []
 431     for line in output:
 432         changeNum = line.split(" ")[1]
 433         changes.append(int(changeNum))
 434
 435     changes.sort()
 436     return changes
 437
 438 class Command:
 439     def __init__(self):
 440         self.usage = "usage: %prog [options]"
 441         self.needsGit = True
 442
 443 class P4Debug(Command):
 444     def __init__(self):
 445         Command.__init__(self)
 446         self.options = [
 447             optparse.make_option("--verbose", dest="verbose", action="store_true",
 448                                  default=False),
 449             ]
 450         self.description = "A tool to debug the output of p4 -G."
 451         self.needsGit = False
 452         self.verbose = False
 453
 454     def run(self, args):
 455         j = 0
 456         for output in p4CmdList(" ".join(args)):
 457             print 'Element: %d' % j
 458             j += 1
 459             print output
 460         return True
 461
 462 class P4RollBack(Command):
 463     def __init__(self):
 464         Command.__init__(self)
 465         self.options = [
 466             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 467             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 468         ]
 469         self.description = "A tool to debug the multi-branch import. Don't use :)"
 470         self.verbose = False
 471         self.rollbackLocalBranches = False
 472
 473     def run(self, args):
 474         if len(args) != 1:
 475             return False
 476         maxChange = int(args[0])
 477
 478         if "p4ExitCode" in p4Cmd("changes -m 1"):
 479             die("Problems executing p4");
 480
 481         if self.rollbackLocalBranches:
 482             refPrefix = "refs/heads/"
 483             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 484         else:
 485             refPrefix = "refs/remotes/"
 486             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 487
 488         for line in lines:
 489             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 490                 line = line.strip()
 491                 ref = refPrefix + line
 492                 log = extractLogMessageFromGitCommit(ref)
 493                 settings = extractSettingsGitLog(log)
 494
 495                 depotPaths = settings['depot-paths']
 496                 change = settings['change']
 497
 498                 changed = False
 499
 500                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 501                                                            for p in depotPaths]))) == 0:
 502                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 503                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 504                     continue
 505
 506                 while change and int(change) > maxChange:
 507                     changed = True
 508                     if self.verbose:
 509                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 510                     system("git update-ref %s \"%s^\"" % (ref, ref))
 511                     log = extractLogMessageFromGitCommit(ref)
 512                     settings =  extractSettingsGitLog(log)
 513
 514
 515                     depotPaths = settings['depot-paths']
 516                     change = settings['change']
 517
 518                 if changed:
 519                     print "%s rewound to %s" % (ref, change)
 520
 521         return True
 522
 523 class P4Submit(Command):
 524     def __init__(self):
 525         Command.__init__(self)
 526         self.options = [
 527                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 528                 optparse.make_option("--origin", dest="origin"),
 529                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 530         ]
 531         self.description = "Submit changes from git to the perforce depot."
 532         self.usage += " [name of git branch to submit into perforce depot]"
 533         self.interactive = True
 534         self.origin = ""
 535         self.detectRename = False
 536         self.verbose = False
 537         self.isWindows = (platform.system() == "Windows")
 538
 539     def check(self):
 540         if len(p4CmdList("opened ...")) > 0:
 541             die("You have files opened with perforce! Close them before starting the sync.")
 542
 543     # replaces everything between 'Description:' and the next P4 submit template field with the
 544     # commit message
 545     def prepareLogMessage(self, template, message):
 546         result = ""
 547
 548         inDescriptionSection = False
 549
 550         for line in template.split("\n"):
 551             if line.startswith("#"):
 552                 result += line + "\n"
 553                 continue
 554
 555             if inDescriptionSection:
 556                 if line.startswith("Files:"):
 557                     inDescriptionSection = False
 558                 else:
 559                     continue
 560             else:
 561                 if line.startswith("Description:"):
 562                     inDescriptionSection = True
 563                     line += "\n"
 564                     for messageLine in message.split("\n"):
 565                         line += "\t" + messageLine + "\n"
 566
 567             result += line + "\n"
 568
 569         return result
 570
 571     def prepareSubmitTemplate(self):
 572         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 573         template = ""
 574         inFilesSection = False
 575         for line in p4_read_pipe_lines("change -o"):
 576             if line.endswith("\r\n"):
 577                 line = line[:-2] + "\n"
 578             if inFilesSection:
 579                 if line.startswith("\t"):
 580                     # path starts and ends with a tab
 581                     path = line[1:]
 582                     lastTab = path.rfind("\t")
 583                     if lastTab != -1:
 584                         path = path[:lastTab]
 585                         if not path.startswith(self.depotPath):
 586                             continue
 587                 else:
 588                     inFilesSection = False
 589             else:
 590                 if line.startswith("Files:"):
 591                     inFilesSection = True
 592
 593             template += line
 594
 595         return template
 596
 597     def applyCommit(self, id):
 598         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 599         diffOpts = ("", "-M")[self.detectRename]
 600         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 601         filesToAdd = set()
 602         filesToDelete = set()
 603         editedFiles = set()
 604         filesToChangeExecBit = {}
 605         for line in diff:
 606             diff = parseDiffTreeEntry(line)
 607             modifier = diff['status']
 608             path = diff['src']
 609             if modifier == "M":
 610                 p4_system("edit \"%s\"" % path)
 611                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 612                     filesToChangeExecBit[path] = diff['dst_mode']
 613                 editedFiles.add(path)
 614             elif modifier == "A":
 615                 filesToAdd.add(path)
 616                 filesToChangeExecBit[path] = diff['dst_mode']
 617                 if path in filesToDelete:
 618                     filesToDelete.remove(path)
 619             elif modifier == "D":
 620                 filesToDelete.add(path)
 621                 if path in filesToAdd:
 622                     filesToAdd.remove(path)
 623             elif modifier == "R":
 624                 src, dest = diff['src'], diff['dst']
 625                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 626                 p4_system("edit \"%s\"" % (dest))
 627                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 628                     filesToChangeExecBit[dest] = diff['dst_mode']
 629                 os.unlink(dest)
 630                 editedFiles.add(dest)
 631                 filesToDelete.add(src)
 632             else:
 633                 die("unknown modifier %s for %s" % (modifier, path))
 634
 635         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 636         patchcmd = diffcmd + " | git apply "
 637         tryPatchCmd = patchcmd + "--check -"
 638         applyPatchCmd = patchcmd + "--check --apply -"
 639
 640         if os.system(tryPatchCmd) != 0:
 641             print "Unfortunately applying the change failed!"
 642             print "What do you want to do?"
 643             response = "x"
 644             while response != "s" and response != "a" and response != "w":
 645                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 646                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 647             if response == "s":
 648                 print "Skipping! Good luck with the next patches..."
 649                 for f in editedFiles:
 650                     p4_system("revert \"%s\"" % f);
 651                 for f in filesToAdd:
 652                     system("rm %s" %f)
 653                 return
 654             elif response == "a":
 655                 os.system(applyPatchCmd)
 656                 if len(filesToAdd) > 0:
 657                     print "You may also want to call p4 add on the following files:"
 658                     print " ".join(filesToAdd)
 659                 if len(filesToDelete):
 660                     print "The following files should be scheduled for deletion with p4 delete:"
 661                     print " ".join(filesToDelete)
 662                 die("Please resolve and submit the conflict manually and "
 663                     + "continue afterwards with git-p4 submit --continue")
 664             elif response == "w":
 665                 system(diffcmd + " > patch.txt")
 666                 print "Patch saved to patch.txt in %s !" % self.clientPath
 667                 die("Please resolve and submit the conflict manually and "
 668                     "continue afterwards with git-p4 submit --continue")
 669
 670         system(applyPatchCmd)
 671
 672         for f in filesToAdd:
 673             p4_system("add \"%s\"" % f)
 674         for f in filesToDelete:
 675             p4_system("revert \"%s\"" % f)
 676             p4_system("delete \"%s\"" % f)
 677
 678         # Set/clear executable bits
 679         for f in filesToChangeExecBit.keys():
 680             mode = filesToChangeExecBit[f]
 681             setP4ExecBit(f, mode)
 682
 683         logMessage = extractLogMessageFromGitCommit(id)
 684         logMessage = logMessage.strip()
 685
 686         template = self.prepareSubmitTemplate()
 687
 688         if self.interactive:
 689             submitTemplate = self.prepareLogMessage(template, logMessage)
 690             if os.environ.has_key("P4DIFF"):
 691                 del(os.environ["P4DIFF"])
 692             diff = p4_read_pipe("diff -du ...")
 693
 694             newdiff = ""
 695             for newFile in filesToAdd:
 696                 newdiff += "==== new file ====\n"
 697                 newdiff += "--- /dev/null\n"
 698                 newdiff += "+++ %s\n" % newFile
 699                 f = open(newFile, "r")
 700                 for line in f.readlines():
 701                     newdiff += "+" + line
 702                 f.close()
 703
 704             separatorLine = "######## everything below this line is just the diff #######\n"
 705
 706             [handle, fileName] = tempfile.mkstemp()
 707             tmpFile = os.fdopen(handle, "w+")
 708             if self.isWindows:
 709                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 710                 separatorLine = separatorLine.replace("\n", "\r\n")
 711                 newdiff = newdiff.replace("\n", "\r\n")
 712             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 713             tmpFile.close()
 714             mtime = os.stat(fileName).st_mtime
 715             defaultEditor = "vi"
 716             if platform.system() == "Windows":
 717                 defaultEditor = "notepad"
 718             if os.environ.has_key("P4EDITOR"):
 719                 editor = os.environ.get("P4EDITOR")
 720             else:
 721                 editor = os.environ.get("EDITOR", defaultEditor);
 722             system(editor + " " + fileName)
 723
 724             response = "y"
 725             if os.stat(fileName).st_mtime <= mtime:
 726                 response = "x"
 727                 while response != "y" and response != "n":
 728                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 729
 730             if response == "y":
 731                 tmpFile = open(fileName, "rb")
 732                 message = tmpFile.read()
 733                 tmpFile.close()
 734                 submitTemplate = message[:message.index(separatorLine)]
 735                 if self.isWindows:
 736                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 737                 p4_write_pipe("submit -i", submitTemplate)
 738             else:
 739                 for f in editedFiles:
 740                     p4_system("revert \"%s\"" % f);
 741                 for f in filesToAdd:
 742                     p4_system("revert \"%s\"" % f);
 743                     system("rm %s" %f)
 744
 745             os.remove(fileName)
 746         else:
 747             fileName = "submit.txt"
 748             file = open(fileName, "w+")
 749             file.write(self.prepareLogMessage(template, logMessage))
 750             file.close()
 751             print ("Perforce submit template written as %s. "
 752                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 753                    % (fileName, fileName))
 754
 755     def run(self, args):
 756         if len(args) == 0:
 757             self.master = currentGitBranch()
 758             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 759                 die("Detecting current git branch failed!")
 760         elif len(args) == 1:
 761             self.master = args[0]
 762         else:
 763             return False
 764
 765         allowSubmit = gitConfig("git-p4.allowSubmit")
 766         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 767             die("%s is not in git-p4.allowSubmit" % self.master)
 768
 769         [upstream, settings] = findUpstreamBranchPoint()
 770         self.depotPath = settings['depot-paths'][0]
 771         if len(self.origin) == 0:
 772             self.origin = upstream
 773
 774         if self.verbose:
 775             print "Origin branch is " + self.origin
 776
 777         if len(self.depotPath) == 0:
 778             print "Internal error: cannot locate perforce depot path from existing branches"
 779             sys.exit(128)
 780
 781         self.clientPath = p4Where(self.depotPath)
 782
 783         if len(self.clientPath) == 0:
 784             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 785             sys.exit(128)
 786
 787         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 788         self.oldWorkingDirectory = os.getcwd()
 789
 790         chdir(self.clientPath)
 791         print "Syncronizing p4 checkout..."
 792         p4_system("sync ...")
 793
 794         self.check()
 795
 796         commits = []
 797         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 798             commits.append(line.strip())
 799         commits.reverse()
 800
 801         while len(commits) > 0:
 802             commit = commits[0]
 803             commits = commits[1:]
 804             self.applyCommit(commit)
 805             if not self.interactive:
 806                 break
 807
 808         if len(commits) == 0:
 809             print "All changes applied!"
 810             chdir(self.oldWorkingDirectory)
 811
 812             sync = P4Sync()
 813             sync.run([])
 814
 815             rebase = P4Rebase()
 816             rebase.rebase()
 817
 818         return True
 819
 820 class P4Sync(Command):
 821     def __init__(self):
 822         Command.__init__(self)
 823         self.options = [
 824                 optparse.make_option("--branch", dest="branch"),
 825                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 826                 optparse.make_option("--changesfile", dest="changesFile"),
 827                 optparse.make_option("--silent", dest="silent", action="store_true"),
 828                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 829                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 830                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 831                                      help="Import into refs/heads/ , not refs/remotes"),
 832                 optparse.make_option("--max-changes", dest="maxChanges"),
 833                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 834                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 835                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 836                                      help="Only sync files that are included in the Perforce Client Spec")
 837         ]
 838         self.description = """Imports from Perforce into a git repository.\n
 839     example:
 840     //depot/my/project/ -- to import the current head
 841     //depot/my/project/@all -- to import everything
 842     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 843
 844     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 845
 846         self.usage += " //depot/path[@revRange]"
 847         self.silent = False
 848         self.createdBranches = Set()
 849         self.committedChanges = Set()
 850         self.branch = ""
 851         self.detectBranches = False
 852         self.detectLabels = False
 853         self.changesFile = ""
 854         self.syncWithOrigin = True
 855         self.verbose = False
 856         self.importIntoRemotes = True
 857         self.maxChanges = ""
 858         self.isWindows = (platform.system() == "Windows")
 859         self.keepRepoPath = False
 860         self.depotPaths = None
 861         self.p4BranchesInGit = []
 862         self.cloneExclude = []
 863         self.useClientSpec = False
 864         self.clientSpecDirs = []
 865
 866         if gitConfig("git-p4.syncFromOrigin") == "false":
 867             self.syncWithOrigin = False
 868
 869     def extractFilesFromCommit(self, commit):
 870         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 871                              for path in self.cloneExclude]
 872         files = []
 873         fnum = 0
 874         while commit.has_key("depotFile%s" % fnum):
 875             path =  commit["depotFile%s" % fnum]
 876
 877             if [p for p in self.cloneExclude
 878                 if path.startswith (p)]:
 879                 found = False
 880             else:
 881                 found = [p for p in self.depotPaths
 882                          if path.startswith (p)]
 883             if not found:
 884                 fnum = fnum + 1
 885                 continue
 886
 887             file = {}
 888             file["path"] = path
 889             file["rev"] = commit["rev%s" % fnum]
 890             file["action"] = commit["action%s" % fnum]
 891             file["type"] = commit["type%s" % fnum]
 892             files.append(file)
 893             fnum = fnum + 1
 894         return files
 895
 896     def stripRepoPath(self, path, prefixes):
 897         if self.keepRepoPath:
 898             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 899
 900         for p in prefixes:
 901             if path.startswith(p):
 902                 path = path[len(p):]
 903
 904         return path
 905
 906     def splitFilesIntoBranches(self, commit):
 907         branches = {}
 908         fnum = 0
 909         while commit.has_key("depotFile%s" % fnum):
 910             path =  commit["depotFile%s" % fnum]
 911             found = [p for p in self.depotPaths
 912                      if path.startswith (p)]
 913             if not found:
 914                 fnum = fnum + 1
 915                 continue
 916
 917             file = {}
 918             file["path"] = path
 919             file["rev"] = commit["rev%s" % fnum]
 920             file["action"] = commit["action%s" % fnum]
 921             file["type"] = commit["type%s" % fnum]
 922             fnum = fnum + 1
 923
 924             relPath = self.stripRepoPath(path, self.depotPaths)
 925
 926             for branch in self.knownBranches.keys():
 927
 928                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 929                 if relPath.startswith(branch + "/"):
 930                     if branch not in branches:
 931                         branches[branch] = []
 932                     branches[branch].append(file)
 933                     break
 934
 935         return branches
 936
 937     ## Should move this out, doesn't use SELF.
 938     def readP4Files(self, files):
 939         filesForCommit = []
 940         filesToRead = []
 941
 942         for f in files:
 943             includeFile = True
 944             for val in self.clientSpecDirs:
 945                 if f['path'].startswith(val[0]):
 946                     if val[1] <= 0:
 947                         includeFile = False
 948                     break
 949
 950             if includeFile:
 951                 filesForCommit.append(f)
 952                 if f['action'] not in ('delete', 'purge'):
 953                     filesToRead.append(f)
 954
 955         filedata = []
 956         if len(filesToRead) > 0:
 957             filedata = p4CmdList('-x - print',
 958                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 959                                                   for f in filesToRead]),
 960                                  stdin_mode='w+')
 961
 962             if "p4ExitCode" in filedata[0]:
 963                 die("Problems executing p4. Error: [%d]."
 964                     % (filedata[0]['p4ExitCode']));
 965
 966         j = 0;
 967         contents = {}
 968         while j < len(filedata):
 969             stat = filedata[j]
 970             j += 1
 971             text = ''
 972             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 973                 text += filedata[j]['data']
 974                 del filedata[j]['data']
 975                 j += 1
 976
 977             if not stat.has_key('depotFile'):
 978                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 979                 continue
 980
 981             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 982                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 983             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 984                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text)
 985
 986             contents[stat['depotFile']] = text
 987
 988         for f in filesForCommit:
 989             path = f['path']
 990             if contents.has_key(path):
 991                 f['data'] = contents[path]
 992
 993         return filesForCommit
 994
 995     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 996         epoch = details["time"]
 997         author = details["user"]
 998
 999         if self.verbose:
1000             print "commit into %s" % branch
1001
1002         # start with reading files; if that fails, we should not
1003         # create a commit.
1004         new_files = []
1005         for f in files:
1006             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1007                 new_files.append (f)
1008             else:
1009                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1010         files = self.readP4Files(new_files)
1011
1012         self.gitStream.write("commit %s\n" % branch)
1013 #        gitStream.write("mark :%s\n" % details["change"])
1014         self.committedChanges.add(int(details["change"]))
1015         committer = ""
1016         if author not in self.users:
1017             self.getUserMapFromPerforceServer()
1018         if author in self.users:
1019             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1020         else:
1021             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1022
1023         self.gitStream.write("committer %s\n" % committer)
1024
1025         self.gitStream.write("data <<EOT\n")
1026         self.gitStream.write(details["desc"])
1027         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1028                              % (','.join (branchPrefixes), details["change"]))
1029         if len(details['options']) > 0:
1030             self.gitStream.write(": options = %s" % details['options'])
1031         self.gitStream.write("]\nEOT\n\n")
1032
1033         if len(parent) > 0:
1034             if self.verbose:
1035                 print "parent %s" % parent
1036             self.gitStream.write("from %s\n" % parent)
1037
1038         for file in files:
1039             if file["type"] == "apple":
1040                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1041                 continue
1042
1043             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1044             if file["action"] in ("delete", "purge"):
1045                 self.gitStream.write("D %s\n" % relPath)
1046             else:
1047                 data = file['data']
1048
1049                 mode = "644"
1050                 if isP4Exec(file["type"]):
1051                     mode = "755"
1052                 elif file["type"] == "symlink":
1053                     mode = "120000"
1054                     # p4 print on a symlink contains "target\n", so strip it off
1055                     data = data[:-1]
1056
1057                 if self.isWindows and file["type"].endswith("text"):
1058                     data = data.replace("\r\n", "\n")
1059
1060                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1061                 self.gitStream.write("data %s\n" % len(data))
1062                 self.gitStream.write(data)
1063                 self.gitStream.write("\n")
1064
1065         self.gitStream.write("\n")
1066
1067         change = int(details["change"])
1068
1069         if self.labels.has_key(change):
1070             label = self.labels[change]
1071             labelDetails = label[0]
1072             labelRevisions = label[1]
1073             if self.verbose:
1074                 print "Change %s is labelled %s" % (change, labelDetails)
1075
1076             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1077                                                     for p in branchPrefixes]))
1078
1079             if len(files) == len(labelRevisions):
1080
1081                 cleanedFiles = {}
1082                 for info in files:
1083                     if info["action"] in ("delete", "purge"):
1084                         continue
1085                     cleanedFiles[info["depotFile"]] = info["rev"]
1086
1087                 if cleanedFiles == labelRevisions:
1088                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1089                     self.gitStream.write("from %s\n" % branch)
1090
1091                     owner = labelDetails["Owner"]
1092                     tagger = ""
1093                     if author in self.users:
1094                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1095                     else:
1096                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1097                     self.gitStream.write("tagger %s\n" % tagger)
1098                     self.gitStream.write("data <<EOT\n")
1099                     self.gitStream.write(labelDetails["Description"])
1100                     self.gitStream.write("EOT\n\n")
1101
1102                 else:
1103                     if not self.silent:
1104                         print ("Tag %s does not match with change %s: files do not match."
1105                                % (labelDetails["label"], change))
1106
1107             else:
1108                 if not self.silent:
1109                     print ("Tag %s does not match with change %s: file count is different."
1110                            % (labelDetails["label"], change))
1111
1112     def getUserCacheFilename(self):
1113         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1114         return home + "/.gitp4-usercache.txt"
1115
1116     def getUserMapFromPerforceServer(self):
1117         if self.userMapFromPerforceServer:
1118             return
1119         self.users = {}
1120
1121         for output in p4CmdList("users"):
1122             if not output.has_key("User"):
1123                 continue
1124             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1125
1126
1127         s = ''
1128         for (key, val) in self.users.items():
1129             s += "%s\t%s\n" % (key, val)
1130
1131         open(self.getUserCacheFilename(), "wb").write(s)
1132         self.userMapFromPerforceServer = True
1133
1134     def loadUserMapFromCache(self):
1135         self.users = {}
1136         self.userMapFromPerforceServer = False
1137         try:
1138             cache = open(self.getUserCacheFilename(), "rb")
1139             lines = cache.readlines()
1140             cache.close()
1141             for line in lines:
1142                 entry = line.strip().split("\t")
1143                 self.users[entry[0]] = entry[1]
1144         except IOError:
1145             self.getUserMapFromPerforceServer()
1146
1147     def getLabels(self):
1148         self.labels = {}
1149
1150         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1151         if len(l) > 0 and not self.silent:
1152             print "Finding files belonging to labels in %s" % `self.depotPaths`
1153
1154         for output in l:
1155             label = output["label"]
1156             revisions = {}
1157             newestChange = 0
1158             if self.verbose:
1159                 print "Querying files for label %s" % label
1160             for file in p4CmdList("files "
1161                                   +  ' '.join (["%s...@%s" % (p, label)
1162                                                 for p in self.depotPaths])):
1163                 revisions[file["depotFile"]] = file["rev"]
1164                 change = int(file["change"])
1165                 if change > newestChange:
1166                     newestChange = change
1167
1168             self.labels[newestChange] = [output, revisions]
1169
1170         if self.verbose:
1171             print "Label changes: %s" % self.labels.keys()
1172
1173     def guessProjectName(self):
1174         for p in self.depotPaths:
1175             if p.endswith("/"):
1176                 p = p[:-1]
1177             p = p[p.strip().rfind("/") + 1:]
1178             if not p.endswith("/"):
1179                p += "/"
1180             return p
1181
1182     def getBranchMapping(self):
1183         lostAndFoundBranches = set()
1184
1185         for info in p4CmdList("branches"):
1186             details = p4Cmd("branch -o %s" % info["branch"])
1187             viewIdx = 0
1188             while details.has_key("View%s" % viewIdx):
1189                 paths = details["View%s" % viewIdx].split(" ")
1190                 viewIdx = viewIdx + 1
1191                 # require standard //depot/foo/... //depot/bar/... mapping
1192                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1193                     continue
1194                 source = paths[0]
1195                 destination = paths[1]
1196                 ## HACK
1197                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1198                     source = source[len(self.depotPaths[0]):-4]
1199                     destination = destination[len(self.depotPaths[0]):-4]
1200
1201                     if destination in self.knownBranches:
1202                         if not self.silent:
1203                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1204                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1205                         continue
1206
1207                     self.knownBranches[destination] = source
1208
1209                     lostAndFoundBranches.discard(destination)
1210
1211                     if source not in self.knownBranches:
1212                         lostAndFoundBranches.add(source)
1213
1214
1215         for branch in lostAndFoundBranches:
1216             self.knownBranches[branch] = branch
1217
1218     def getBranchMappingFromGitBranches(self):
1219         branches = p4BranchesInGit(self.importIntoRemotes)
1220         for branch in branches.keys():
1221             if branch == "master":
1222                 branch = "main"
1223             else:
1224                 branch = branch[len(self.projectName):]
1225             self.knownBranches[branch] = branch
1226
1227     def listExistingP4GitBranches(self):
1228         # branches holds mapping from name to commit
1229         branches = p4BranchesInGit(self.importIntoRemotes)
1230         self.p4BranchesInGit = branches.keys()
1231         for branch in branches.keys():
1232             self.initialParents[self.refPrefix + branch] = branches[branch]
1233
1234     def updateOptionDict(self, d):
1235         option_keys = {}
1236         if self.keepRepoPath:
1237             option_keys['keepRepoPath'] = 1
1238
1239         d["options"] = ' '.join(sorted(option_keys.keys()))
1240
1241     def readOptions(self, d):
1242         self.keepRepoPath = (d.has_key('options')
1243                              and ('keepRepoPath' in d['options']))
1244
1245     def gitRefForBranch(self, branch):
1246         if branch == "main":
1247             return self.refPrefix + "master"
1248
1249         if len(branch) <= 0:
1250             return branch
1251
1252         return self.refPrefix + self.projectName + branch
1253
1254     def gitCommitByP4Change(self, ref, change):
1255         if self.verbose:
1256             print "looking in ref " + ref + " for change %s using bisect..." % change
1257
1258         earliestCommit = ""
1259         latestCommit = parseRevision(ref)
1260
1261         while True:
1262             if self.verbose:
1263                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1264             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1265             if len(next) == 0:
1266                 if self.verbose:
1267                     print "argh"
1268                 return ""
1269             log = extractLogMessageFromGitCommit(next)
1270             settings = extractSettingsGitLog(log)
1271             currentChange = int(settings['change'])
1272             if self.verbose:
1273                 print "current change %s" % currentChange
1274
1275             if currentChange == change:
1276                 if self.verbose:
1277                     print "found %s" % next
1278                 return next
1279
1280             if currentChange < change:
1281                 earliestCommit = "^%s" % next
1282             else:
1283                 latestCommit = "%s" % next
1284
1285         return ""
1286
1287     def importNewBranch(self, branch, maxChange):
1288         # make fast-import flush all changes to disk and update the refs using the checkpoint
1289         # command so that we can try to find the branch parent in the git history
1290         self.gitStream.write("checkpoint\n\n");
1291         self.gitStream.flush();
1292         branchPrefix = self.depotPaths[0] + branch + "/"
1293         range = "@1,%s" % maxChange
1294         #print "prefix" + branchPrefix
1295         changes = p4ChangesForPaths([branchPrefix], range)
1296         if len(changes) <= 0:
1297             return False
1298         firstChange = changes[0]
1299         #print "first change in branch: %s" % firstChange
1300         sourceBranch = self.knownBranches[branch]
1301         sourceDepotPath = self.depotPaths[0] + sourceBranch
1302         sourceRef = self.gitRefForBranch(sourceBranch)
1303         #print "source " + sourceBranch
1304
1305         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1306         #print "branch parent: %s" % branchParentChange
1307         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1308         if len(gitParent) > 0:
1309             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1310             #print "parent git commit: %s" % gitParent
1311
1312         self.importChanges(changes)
1313         return True
1314
1315     def importChanges(self, changes):
1316         cnt = 1
1317         for change in changes:
1318             description = p4Cmd("describe %s" % change)
1319             self.updateOptionDict(description)
1320
1321             if not self.silent:
1322                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1323                 sys.stdout.flush()
1324             cnt = cnt + 1
1325
1326             try:
1327                 if self.detectBranches:
1328                     branches = self.splitFilesIntoBranches(description)
1329                     for branch in branches.keys():
1330                         ## HACK  --hwn
1331                         branchPrefix = self.depotPaths[0] + branch + "/"
1332
1333                         parent = ""
1334
1335                         filesForCommit = branches[branch]
1336
1337                         if self.verbose:
1338                             print "branch is %s" % branch
1339
1340                         self.updatedBranches.add(branch)
1341
1342                         if branch not in self.createdBranches:
1343                             self.createdBranches.add(branch)
1344                             parent = self.knownBranches[branch]
1345                             if parent == branch:
1346                                 parent = ""
1347                             else:
1348                                 fullBranch = self.projectName + branch
1349                                 if fullBranch not in self.p4BranchesInGit:
1350                                     if not self.silent:
1351                                         print("\n    Importing new branch %s" % fullBranch);
1352                                     if self.importNewBranch(branch, change - 1):
1353                                         parent = ""
1354                                         self.p4BranchesInGit.append(fullBranch)
1355                                     if not self.silent:
1356                                         print("\n    Resuming with change %s" % change);
1357
1358                                 if self.verbose:
1359                                     print "parent determined through known branches: %s" % parent
1360
1361                         branch = self.gitRefForBranch(branch)
1362                         parent = self.gitRefForBranch(parent)
1363
1364                         if self.verbose:
1365                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1366
1367                         if len(parent) == 0 and branch in self.initialParents:
1368                             parent = self.initialParents[branch]
1369                             del self.initialParents[branch]
1370
1371                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1372                 else:
1373                     files = self.extractFilesFromCommit(description)
1374                     self.commit(description, files, self.branch, self.depotPaths,
1375                                 self.initialParent)
1376                     self.initialParent = ""
1377             except IOError:
1378                 print self.gitError.read()
1379                 sys.exit(1)
1380
1381     def importHeadRevision(self, revision):
1382         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1383
1384         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1385         details["desc"] = ("Initial import of %s from the state at revision %s"
1386                            % (' '.join(self.depotPaths), revision))
1387         details["change"] = revision
1388         newestRevision = 0
1389
1390         fileCnt = 0
1391         for info in p4CmdList("files "
1392                               +  ' '.join(["%s...%s"
1393                                            % (p, revision)
1394                                            for p in self.depotPaths])):
1395
1396             if info['code'] == 'error':
1397                 sys.stderr.write("p4 returned an error: %s\n"
1398                                  % info['data'])
1399                 sys.exit(1)
1400
1401
1402             change = int(info["change"])
1403             if change > newestRevision:
1404                 newestRevision = change
1405
1406             if info["action"] in ("delete", "purge"):
1407                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1408                 #fileCnt = fileCnt + 1
1409                 continue
1410
1411             for prop in ["depotFile", "rev", "action", "type" ]:
1412                 details["%s%s" % (prop, fileCnt)] = info[prop]
1413
1414             fileCnt = fileCnt + 1
1415
1416         details["change"] = newestRevision
1417         self.updateOptionDict(details)
1418         try:
1419             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1420         except IOError:
1421             print "IO error with git fast-import. Is your git version recent enough?"
1422             print self.gitError.read()
1423
1424
1425     def getClientSpec(self):
1426         specList = p4CmdList( "client -o" )
1427         temp = {}
1428         for entry in specList:
1429             for k,v in entry.iteritems():
1430                 if k.startswith("View"):
1431                     if v.startswith('"'):
1432                         start = 1
1433                     else:
1434                         start = 0
1435                     index = v.find("...")
1436                     v = v[start:index]
1437                     if v.startswith("-"):
1438                         v = v[1:]
1439                         temp[v] = -len(v)
1440                     else:
1441                         temp[v] = len(v)
1442         self.clientSpecDirs = temp.items()
1443         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1444
1445     def run(self, args):
1446         self.depotPaths = []
1447         self.changeRange = ""
1448         self.initialParent = ""
1449         self.previousDepotPaths = []
1450
1451         # map from branch depot path to parent branch
1452         self.knownBranches = {}
1453         self.initialParents = {}
1454         self.hasOrigin = originP4BranchesExist()
1455         if not self.syncWithOrigin:
1456             self.hasOrigin = False
1457
1458         if self.importIntoRemotes:
1459             self.refPrefix = "refs/remotes/p4/"
1460         else:
1461             self.refPrefix = "refs/heads/p4/"
1462
1463         if self.syncWithOrigin and self.hasOrigin:
1464             if not self.silent:
1465                 print "Syncing with origin first by calling git fetch origin"
1466             system("git fetch origin")
1467
1468         if len(self.branch) == 0:
1469             self.branch = self.refPrefix + "master"
1470             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1471                 system("git update-ref %s refs/heads/p4" % self.branch)
1472                 system("git branch -D p4");
1473             # create it /after/ importing, when master exists
1474             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1475                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1476
1477         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1478             self.getClientSpec()
1479
1480         # TODO: should always look at previous commits,
1481         # merge with previous imports, if possible.
1482         if args == []:
1483             if self.hasOrigin:
1484                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1485             self.listExistingP4GitBranches()
1486
1487             if len(self.p4BranchesInGit) > 1:
1488                 if not self.silent:
1489                     print "Importing from/into multiple branches"
1490                 self.detectBranches = True
1491
1492             if self.verbose:
1493                 print "branches: %s" % self.p4BranchesInGit
1494
1495             p4Change = 0
1496             for branch in self.p4BranchesInGit:
1497                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1498
1499                 settings = extractSettingsGitLog(logMsg)
1500
1501                 self.readOptions(settings)
1502                 if (settings.has_key('depot-paths')
1503                     and settings.has_key ('change')):
1504                     change = int(settings['change']) + 1
1505                     p4Change = max(p4Change, change)
1506
1507                     depotPaths = sorted(settings['depot-paths'])
1508                     if self.previousDepotPaths == []:
1509                         self.previousDepotPaths = depotPaths
1510                     else:
1511                         paths = []
1512                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1513                             for i in range(0, min(len(cur), len(prev))):
1514                                 if cur[i] <> prev[i]:
1515                                     i = i - 1
1516                                     break
1517
1518                             paths.append (cur[:i + 1])
1519
1520                         self.previousDepotPaths = paths
1521
1522             if p4Change > 0:
1523                 self.depotPaths = sorted(self.previousDepotPaths)
1524                 self.changeRange = "@%s,#head" % p4Change
1525                 if not self.detectBranches:
1526                     self.initialParent = parseRevision(self.branch)
1527                 if not self.silent and not self.detectBranches:
1528                     print "Performing incremental import into %s git branch" % self.branch
1529
1530         if not self.branch.startswith("refs/"):
1531             self.branch = "refs/heads/" + self.branch
1532
1533         if len(args) == 0 and self.depotPaths:
1534             if not self.silent:
1535                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1536         else:
1537             if self.depotPaths and self.depotPaths != args:
1538                 print ("previous import used depot path %s and now %s was specified. "
1539                        "This doesn't work!" % (' '.join (self.depotPaths),
1540                                                ' '.join (args)))
1541                 sys.exit(1)
1542
1543             self.depotPaths = sorted(args)
1544
1545         revision = ""
1546         self.users = {}
1547
1548         newPaths = []
1549         for p in self.depotPaths:
1550             if p.find("@") != -1:
1551                 atIdx = p.index("@")
1552                 self.changeRange = p[atIdx:]
1553                 if self.changeRange == "@all":
1554                     self.changeRange = ""
1555                 elif ',' not in self.changeRange:
1556                     revision = self.changeRange
1557                     self.changeRange = ""
1558                 p = p[:atIdx]
1559             elif p.find("#") != -1:
1560                 hashIdx = p.index("#")
1561                 revision = p[hashIdx:]
1562                 p = p[:hashIdx]
1563             elif self.previousDepotPaths == []:
1564                 revision = "#head"
1565
1566             p = re.sub ("\.\.\.$", "", p)
1567             if not p.endswith("/"):
1568                 p += "/"
1569
1570             newPaths.append(p)
1571
1572         self.depotPaths = newPaths
1573
1574
1575         self.loadUserMapFromCache()
1576         self.labels = {}
1577         if self.detectLabels:
1578             self.getLabels();
1579
1580         if self.detectBranches:
1581             ## FIXME - what's a P4 projectName ?
1582             self.projectName = self.guessProjectName()
1583
1584             if self.hasOrigin:
1585                 self.getBranchMappingFromGitBranches()
1586             else:
1587                 self.getBranchMapping()
1588             if self.verbose:
1589                 print "p4-git branches: %s" % self.p4BranchesInGit
1590                 print "initial parents: %s" % self.initialParents
1591             for b in self.p4BranchesInGit:
1592                 if b != "master":
1593
1594                     ## FIXME
1595                     b = b[len(self.projectName):]
1596                 self.createdBranches.add(b)
1597
1598         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1599
1600         importProcess = subprocess.Popen(["git", "fast-import"],
1601                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1602                                          stderr=subprocess.PIPE);
1603         self.gitOutput = importProcess.stdout
1604         self.gitStream = importProcess.stdin
1605         self.gitError = importProcess.stderr
1606
1607         if revision:
1608             self.importHeadRevision(revision)
1609         else:
1610             changes = []
1611
1612             if len(self.changesFile) > 0:
1613                 output = open(self.changesFile).readlines()
1614                 changeSet = Set()
1615                 for line in output:
1616                     changeSet.add(int(line))
1617
1618                 for change in changeSet:
1619                     changes.append(change)
1620
1621                 changes.sort()
1622             else:
1623                 if self.verbose:
1624                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1625                                                               self.changeRange)
1626                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1627
1628                 if len(self.maxChanges) > 0:
1629                     changes = changes[:min(int(self.maxChanges), len(changes))]
1630
1631             if len(changes) == 0:
1632                 if not self.silent:
1633                     print "No changes to import!"
1634                 return True
1635
1636             if not self.silent and not self.detectBranches:
1637                 print "Import destination: %s" % self.branch
1638
1639             self.updatedBranches = set()
1640
1641             self.importChanges(changes)
1642
1643             if not self.silent:
1644                 print ""
1645                 if len(self.updatedBranches) > 0:
1646                     sys.stdout.write("Updated branches: ")
1647                     for b in self.updatedBranches:
1648                         sys.stdout.write("%s " % b)
1649                     sys.stdout.write("\n")
1650
1651         self.gitStream.close()
1652         if importProcess.wait() != 0:
1653             die("fast-import failed: %s" % self.gitError.read())
1654         self.gitOutput.close()
1655         self.gitError.close()
1656
1657         return True
1658
1659 class P4Rebase(Command):
1660     def __init__(self):
1661         Command.__init__(self)
1662         self.options = [ ]
1663         self.description = ("Fetches the latest revision from perforce and "
1664                             + "rebases the current work (branch) against it")
1665         self.verbose = False
1666
1667     def run(self, args):
1668         sync = P4Sync()
1669         sync.run([])
1670
1671         return self.rebase()
1672
1673     def rebase(self):
1674         if os.system("git update-index --refresh") != 0:
1675             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1676         if len(read_pipe("git diff-index HEAD --")) > 0:
1677             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1678
1679         [upstream, settings] = findUpstreamBranchPoint()
1680         if len(upstream) == 0:
1681             die("Cannot find upstream branchpoint for rebase")
1682
1683         # the branchpoint may be p4/foo~3, so strip off the parent
1684         upstream = re.sub("~[0-9]+$", "", upstream)
1685
1686         print "Rebasing the current branch onto %s" % upstream
1687         oldHead = read_pipe("git rev-parse HEAD").strip()
1688         system("git rebase %s" % upstream)
1689         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1690         return True
1691
1692 class P4Clone(P4Sync):
1693     def __init__(self):
1694         P4Sync.__init__(self)
1695         self.description = "Creates a new git repository and imports from Perforce into it"
1696         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1697         self.options += [
1698             optparse.make_option("--destination", dest="cloneDestination",
1699                                  action='store', default=None,
1700                                  help="where to leave result of the clone"),
1701             optparse.make_option("-/", dest="cloneExclude",
1702                                  action="append", type="string",
1703                                  help="exclude depot path")
1704         ]
1705         self.cloneDestination = None
1706         self.needsGit = False
1707
1708     # This is required for the "append" cloneExclude action
1709     def ensure_value(self, attr, value):
1710         if not hasattr(self, attr) or getattr(self, attr) is None:
1711             setattr(self, attr, value)
1712         return getattr(self, attr)
1713
1714     def defaultDestination(self, args):
1715         ## TODO: use common prefix of args?
1716         depotPath = args[0]
1717         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1718         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1719         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1720         depotDir = re.sub(r"/$", "", depotDir)
1721         return os.path.split(depotDir)[1]
1722
1723     def run(self, args):
1724         if len(args) < 1:
1725             return False
1726
1727         if self.keepRepoPath and not self.cloneDestination:
1728             sys.stderr.write("Must specify destination for --keep-path\n")
1729             sys.exit(1)
1730
1731         depotPaths = args
1732
1733         if not self.cloneDestination and len(depotPaths) > 1:
1734             self.cloneDestination = depotPaths[-1]
1735             depotPaths = depotPaths[:-1]
1736
1737         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1738         for p in depotPaths:
1739             if not p.startswith("//"):
1740                 return False
1741
1742         if not self.cloneDestination:
1743             self.cloneDestination = self.defaultDestination(args)
1744
1745         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1746         if not os.path.exists(self.cloneDestination):
1747             os.makedirs(self.cloneDestination)
1748         chdir(self.cloneDestination)
1749         system("git init")
1750         self.gitdir = os.getcwd() + "/.git"
1751         if not P4Sync.run(self, depotPaths):
1752             return False
1753         if self.branch != "master":
1754             if self.importIntoRemotes:
1755                 masterbranch = "refs/remotes/p4/master"
1756             else:
1757                 masterbranch = "refs/heads/p4/master"
1758             if gitBranchExists(masterbranch):
1759                 system("git branch master %s" % masterbranch)
1760                 system("git checkout -f")
1761             else:
1762                 print "Could not detect main branch. No checkout/master branch created."
1763
1764         return True
1765
1766 class P4Branches(Command):
1767     def __init__(self):
1768         Command.__init__(self)
1769         self.options = [ ]
1770         self.description = ("Shows the git branches that hold imports and their "
1771                             + "corresponding perforce depot paths")
1772         self.verbose = False
1773
1774     def run(self, args):
1775         if originP4BranchesExist():
1776             createOrUpdateBranchesFromOrigin()
1777
1778         cmdline = "git rev-parse --symbolic "
1779         cmdline += " --remotes"
1780
1781         for line in read_pipe_lines(cmdline):
1782             line = line.strip()
1783
1784             if not line.startswith('p4/') or line == "p4/HEAD":
1785                 continue
1786             branch = line
1787
1788             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1789             settings = extractSettingsGitLog(log)
1790
1791             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1792         return True
1793
1794 class HelpFormatter(optparse.IndentedHelpFormatter):
1795     def __init__(self):
1796         optparse.IndentedHelpFormatter.__init__(self)
1797
1798     def format_description(self, description):
1799         if description:
1800             return description + "\n"
1801         else:
1802             return ""
1803
1804 def printUsage(commands):
1805     print "usage: %s <command> [options]" % sys.argv[0]
1806     print ""
1807     print "valid commands: %s" % ", ".join(commands)
1808     print ""
1809     print "Try %s <command> --help for command specific help." % sys.argv[0]
1810     print ""
1811
1812 commands = {
1813     "debug" : P4Debug,
1814     "submit" : P4Submit,
1815     "commit" : P4Submit,
1816     "sync" : P4Sync,
1817     "rebase" : P4Rebase,
1818     "clone" : P4Clone,
1819     "rollback" : P4RollBack,
1820     "branches" : P4Branches
1821 }
1822
1823
1824 def main():
1825     if len(sys.argv[1:]) == 0:
1826         printUsage(commands.keys())
1827         sys.exit(2)
1828
1829     cmd = ""
1830     cmdName = sys.argv[1]
1831     try:
1832         klass = commands[cmdName]
1833         cmd = klass()
1834     except KeyError:
1835         print "unknown command %s" % cmdName
1836         print ""
1837         printUsage(commands.keys())
1838         sys.exit(2)
1839
1840     options = cmd.options
1841     cmd.gitdir = os.environ.get("GIT_DIR", None)
1842
1843     args = sys.argv[2:]
1844
1845     if len(options) > 0:
1846         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1847
1848         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1849                                        options,
1850                                        description = cmd.description,
1851                                        formatter = HelpFormatter())
1852
1853         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1854     global verbose
1855     verbose = cmd.verbose
1856     if cmd.needsGit:
1857         if cmd.gitdir == None:
1858             cmd.gitdir = os.path.abspath(".git")
1859             if not isValidGitDir(cmd.gitdir):
1860                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1861                 if os.path.exists(cmd.gitdir):
1862                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1863                     if len(cdup) > 0:
1864                         chdir(cdup);
1865
1866         if not isValidGitDir(cmd.gitdir):
1867             if isValidGitDir(cmd.gitdir + "/.git"):
1868                 cmd.gitdir += "/.git"
1869             else:
1870                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1871
1872         os.environ["GIT_DIR"] = cmd.gitdir
1873
1874     if not cmd.run(args):
1875         parser.print_help()
1876
1877
1878 if __name__ == '__main__':
1879     main()