contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def die(msg):
  55     if verbose:
  56         raise Exception(msg)
  57     else:
  58         sys.stderr.write(msg + "\n")
  59         sys.exit(1)
  60
  61 def write_pipe(c, str):
  62     if verbose:
  63         sys.stderr.write('Writing pipe: %s\n' % c)
  64
  65     pipe = os.popen(c, 'w')
  66     val = pipe.write(str)
  67     if pipe.close():
  68         die('Command failed: %s' % c)
  69
  70     return val
  71
  72 def p4_write_pipe(c, str):
  73     real_cmd = p4_build_cmd(c)
  74     return write_pipe(c, str)
  75
  76 def read_pipe(c, ignore_error=False):
  77     if verbose:
  78         sys.stderr.write('Reading pipe: %s\n' % c)
  79
  80     pipe = os.popen(c, 'rb')
  81     val = pipe.read()
  82     if pipe.close() and not ignore_error:
  83         die('Command failed: %s' % c)
  84
  85     return val
  86
  87 def p4_read_pipe(c, ignore_error=False):
  88     real_cmd = p4_build_cmd(c)
  89     return read_pipe(real_cmd, ignore_error)
  90
  91 def read_pipe_lines(c):
  92     if verbose:
  93         sys.stderr.write('Reading pipe: %s\n' % c)
  94     ## todo: check return status
  95     pipe = os.popen(c, 'rb')
  96     val = pipe.readlines()
  97     if pipe.close():
  98         die('Command failed: %s' % c)
  99
 100     return val
 101
 102 def p4_read_pipe_lines(c):
 103     """Specifically invoke p4 on the command supplied. """
 104     real_cmd = p4_build_cmd(c)
 105     return read_pipe_lines(real_cmd)
 106
 107 def system(cmd):
 108     if verbose:
 109         sys.stderr.write("executing %s\n" % cmd)
 110     if os.system(cmd) != 0:
 111         die("command failed: %s" % cmd)
 112
 113 def p4_system(cmd):
 114     """Specifically invoke p4 as the system command. """
 115     real_cmd = p4_build_cmd(cmd)
 116     return system(real_cmd)
 117
 118 def isP4Exec(kind):
 119     """Determine if a Perforce 'kind' should have execute permission
 120
 121     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 122     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 123     a plus sign, it is also executable"""
 124     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 125
 126 def setP4ExecBit(file, mode):
 127     # Reopens an already open file and changes the execute bit to match
 128     # the execute bit setting in the passed in mode.
 129
 130     p4Type = "+x"
 131
 132     if not isModeExec(mode):
 133         p4Type = getP4OpenedType(file)
 134         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 135         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 136         if p4Type[-1] == "+":
 137             p4Type = p4Type[0:-1]
 138
 139     p4_system("reopen -t %s %s" % (p4Type, file))
 140
 141 def getP4OpenedType(file):
 142     # Returns the perforce file type for the given file.
 143
 144     result = read_pipe("p4 opened %s" % file)
 145     match = re.match(".*\((.+)\)\r?$", result)
 146     if match:
 147         return match.group(1)
 148     else:
 149         die("Could not determine file type for %s (result: '%s')" % (file, result))
 150
 151 def diffTreePattern():
 152     # This is a simple generator for the diff tree regex pattern. This could be
 153     # a class variable if this and parseDiffTreeEntry were a part of a class.
 154     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 155     while True:
 156         yield pattern
 157
 158 def parseDiffTreeEntry(entry):
 159     """Parses a single diff tree entry into its component elements.
 160
 161     See git-diff-tree(1) manpage for details about the format of the diff
 162     output. This method returns a dictionary with the following elements:
 163
 164     src_mode - The mode of the source file
 165     dst_mode - The mode of the destination file
 166     src_sha1 - The sha1 for the source file
 167     dst_sha1 - The sha1 fr the destination file
 168     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 169     status_score - The score for the status (applicable for 'C' and 'R'
 170                    statuses). This is None if there is no score.
 171     src - The path for the source file.
 172     dst - The path for the destination file. This is only present for
 173           copy or renames. If it is not present, this is None.
 174
 175     If the pattern is not matched, None is returned."""
 176
 177     match = diffTreePattern().next().match(entry)
 178     if match:
 179         return {
 180             'src_mode': match.group(1),
 181             'dst_mode': match.group(2),
 182             'src_sha1': match.group(3),
 183             'dst_sha1': match.group(4),
 184             'status': match.group(5),
 185             'status_score': match.group(6),
 186             'src': match.group(7),
 187             'dst': match.group(10)
 188         }
 189     return None
 190
 191 def isModeExec(mode):
 192     # Returns True if the given git mode represents an executable file,
 193     # otherwise False.
 194     return mode[-3:] == "755"
 195
 196 def isModeExecChanged(src_mode, dst_mode):
 197     return isModeExec(src_mode) != isModeExec(dst_mode)
 198
 199 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 200     cmd = p4_build_cmd("-G %s" % (cmd))
 201     if verbose:
 202         sys.stderr.write("Opening pipe: %s\n" % cmd)
 203
 204     # Use a temporary file to avoid deadlocks without
 205     # subprocess.communicate(), which would put another copy
 206     # of stdout into memory.
 207     stdin_file = None
 208     if stdin is not None:
 209         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 210         stdin_file.write(stdin)
 211         stdin_file.flush()
 212         stdin_file.seek(0)
 213
 214     p4 = subprocess.Popen(cmd, shell=True,
 215                           stdin=stdin_file,
 216                           stdout=subprocess.PIPE)
 217
 218     result = []
 219     try:
 220         while True:
 221             entry = marshal.load(p4.stdout)
 222             result.append(entry)
 223     except EOFError:
 224         pass
 225     exitCode = p4.wait()
 226     if exitCode != 0:
 227         entry = {}
 228         entry["p4ExitCode"] = exitCode
 229         result.append(entry)
 230
 231     return result
 232
 233 def p4Cmd(cmd):
 234     list = p4CmdList(cmd)
 235     result = {}
 236     for entry in list:
 237         result.update(entry)
 238     return result;
 239
 240 def p4Where(depotPath):
 241     if not depotPath.endswith("/"):
 242         depotPath += "/"
 243     output = p4Cmd("where %s..." % depotPath)
 244     if output["code"] == "error":
 245         return ""
 246     clientPath = ""
 247     if "path" in output:
 248         clientPath = output.get("path")
 249     elif "data" in output:
 250         data = output.get("data")
 251         lastSpace = data.rfind(" ")
 252         clientPath = data[lastSpace + 1:]
 253
 254     if clientPath.endswith("..."):
 255         clientPath = clientPath[:-3]
 256     return clientPath
 257
 258 def currentGitBranch():
 259     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 260
 261 def isValidGitDir(path):
 262     if (os.path.exists(path + "/HEAD")
 263         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 264         return True;
 265     return False
 266
 267 def parseRevision(ref):
 268     return read_pipe("git rev-parse %s" % ref).strip()
 269
 270 def extractLogMessageFromGitCommit(commit):
 271     logMessage = ""
 272
 273     ## fixme: title is first line of commit, not 1st paragraph.
 274     foundTitle = False
 275     for log in read_pipe_lines("git cat-file commit %s" % commit):
 276        if not foundTitle:
 277            if len(log) == 1:
 278                foundTitle = True
 279            continue
 280
 281        logMessage += log
 282     return logMessage
 283
 284 def extractSettingsGitLog(log):
 285     values = {}
 286     for line in log.split("\n"):
 287         line = line.strip()
 288         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 289         if not m:
 290             continue
 291
 292         assignments = m.group(1).split (':')
 293         for a in assignments:
 294             vals = a.split ('=')
 295             key = vals[0].strip()
 296             val = ('='.join (vals[1:])).strip()
 297             if val.endswith ('\"') and val.startswith('"'):
 298                 val = val[1:-1]
 299
 300             values[key] = val
 301
 302     paths = values.get("depot-paths")
 303     if not paths:
 304         paths = values.get("depot-path")
 305     if paths:
 306         values['depot-paths'] = paths.split(',')
 307     return values
 308
 309 def gitBranchExists(branch):
 310     proc = subprocess.Popen(["git", "rev-parse", branch],
 311                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 312     return proc.wait() == 0;
 313
 314 def gitConfig(key):
 315     return read_pipe("git config %s" % key, ignore_error=True).strip()
 316
 317 def p4BranchesInGit(branchesAreInRemotes = True):
 318     branches = {}
 319
 320     cmdline = "git rev-parse --symbolic "
 321     if branchesAreInRemotes:
 322         cmdline += " --remotes"
 323     else:
 324         cmdline += " --branches"
 325
 326     for line in read_pipe_lines(cmdline):
 327         line = line.strip()
 328
 329         ## only import to p4/
 330         if not line.startswith('p4/') or line == "p4/HEAD":
 331             continue
 332         branch = line
 333
 334         # strip off p4
 335         branch = re.sub ("^p4/", "", line)
 336
 337         branches[branch] = parseRevision(line)
 338     return branches
 339
 340 def findUpstreamBranchPoint(head = "HEAD"):
 341     branches = p4BranchesInGit()
 342     # map from depot-path to branch name
 343     branchByDepotPath = {}
 344     for branch in branches.keys():
 345         tip = branches[branch]
 346         log = extractLogMessageFromGitCommit(tip)
 347         settings = extractSettingsGitLog(log)
 348         if settings.has_key("depot-paths"):
 349             paths = ",".join(settings["depot-paths"])
 350             branchByDepotPath[paths] = "remotes/p4/" + branch
 351
 352     settings = None
 353     parent = 0
 354     while parent < 65535:
 355         commit = head + "~%s" % parent
 356         log = extractLogMessageFromGitCommit(commit)
 357         settings = extractSettingsGitLog(log)
 358         if settings.has_key("depot-paths"):
 359             paths = ",".join(settings["depot-paths"])
 360             if branchByDepotPath.has_key(paths):
 361                 return [branchByDepotPath[paths], settings]
 362
 363         parent = parent + 1
 364
 365     return ["", settings]
 366
 367 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 368     if not silent:
 369         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 370                % localRefPrefix)
 371
 372     originPrefix = "origin/p4/"
 373
 374     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 375         line = line.strip()
 376         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 377             continue
 378
 379         headName = line[len(originPrefix):]
 380         remoteHead = localRefPrefix + headName
 381         originHead = line
 382
 383         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 384         if (not original.has_key('depot-paths')
 385             or not original.has_key('change')):
 386             continue
 387
 388         update = False
 389         if not gitBranchExists(remoteHead):
 390             if verbose:
 391                 print "creating %s" % remoteHead
 392             update = True
 393         else:
 394             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 395             if settings.has_key('change') > 0:
 396                 if settings['depot-paths'] == original['depot-paths']:
 397                     originP4Change = int(original['change'])
 398                     p4Change = int(settings['change'])
 399                     if originP4Change > p4Change:
 400                         print ("%s (%s) is newer than %s (%s). "
 401                                "Updating p4 branch from origin."
 402                                % (originHead, originP4Change,
 403                                   remoteHead, p4Change))
 404                         update = True
 405                 else:
 406                     print ("Ignoring: %s was imported from %s while "
 407                            "%s was imported from %s"
 408                            % (originHead, ','.join(original['depot-paths']),
 409                               remoteHead, ','.join(settings['depot-paths'])))
 410
 411         if update:
 412             system("git update-ref %s %s" % (remoteHead, originHead))
 413
 414 def originP4BranchesExist():
 415         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 416
 417 def p4ChangesForPaths(depotPaths, changeRange):
 418     assert depotPaths
 419     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 420                                                         for p in depotPaths]))
 421
 422     changes = []
 423     for line in output:
 424         changeNum = line.split(" ")[1]
 425         changes.append(int(changeNum))
 426
 427     changes.sort()
 428     return changes
 429
 430 class Command:
 431     def __init__(self):
 432         self.usage = "usage: %prog [options]"
 433         self.needsGit = True
 434
 435 class P4Debug(Command):
 436     def __init__(self):
 437         Command.__init__(self)
 438         self.options = [
 439             optparse.make_option("--verbose", dest="verbose", action="store_true",
 440                                  default=False),
 441             ]
 442         self.description = "A tool to debug the output of p4 -G."
 443         self.needsGit = False
 444         self.verbose = False
 445
 446     def run(self, args):
 447         j = 0
 448         for output in p4CmdList(" ".join(args)):
 449             print 'Element: %d' % j
 450             j += 1
 451             print output
 452         return True
 453
 454 class P4RollBack(Command):
 455     def __init__(self):
 456         Command.__init__(self)
 457         self.options = [
 458             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 459             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 460         ]
 461         self.description = "A tool to debug the multi-branch import. Don't use :)"
 462         self.verbose = False
 463         self.rollbackLocalBranches = False
 464
 465     def run(self, args):
 466         if len(args) != 1:
 467             return False
 468         maxChange = int(args[0])
 469
 470         if "p4ExitCode" in p4Cmd("changes -m 1"):
 471             die("Problems executing p4");
 472
 473         if self.rollbackLocalBranches:
 474             refPrefix = "refs/heads/"
 475             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 476         else:
 477             refPrefix = "refs/remotes/"
 478             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 479
 480         for line in lines:
 481             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 482                 line = line.strip()
 483                 ref = refPrefix + line
 484                 log = extractLogMessageFromGitCommit(ref)
 485                 settings = extractSettingsGitLog(log)
 486
 487                 depotPaths = settings['depot-paths']
 488                 change = settings['change']
 489
 490                 changed = False
 491
 492                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 493                                                            for p in depotPaths]))) == 0:
 494                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 495                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 496                     continue
 497
 498                 while change and int(change) > maxChange:
 499                     changed = True
 500                     if self.verbose:
 501                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 502                     system("git update-ref %s \"%s^\"" % (ref, ref))
 503                     log = extractLogMessageFromGitCommit(ref)
 504                     settings =  extractSettingsGitLog(log)
 505
 506
 507                     depotPaths = settings['depot-paths']
 508                     change = settings['change']
 509
 510                 if changed:
 511                     print "%s rewound to %s" % (ref, change)
 512
 513         return True
 514
 515 class P4Submit(Command):
 516     def __init__(self):
 517         Command.__init__(self)
 518         self.options = [
 519                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 520                 optparse.make_option("--origin", dest="origin"),
 521                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 522         ]
 523         self.description = "Submit changes from git to the perforce depot."
 524         self.usage += " [name of git branch to submit into perforce depot]"
 525         self.interactive = True
 526         self.origin = ""
 527         self.detectRename = False
 528         self.verbose = False
 529         self.isWindows = (platform.system() == "Windows")
 530
 531     def check(self):
 532         if len(p4CmdList("opened ...")) > 0:
 533             die("You have files opened with perforce! Close them before starting the sync.")
 534
 535     # replaces everything between 'Description:' and the next P4 submit template field with the
 536     # commit message
 537     def prepareLogMessage(self, template, message):
 538         result = ""
 539
 540         inDescriptionSection = False
 541
 542         for line in template.split("\n"):
 543             if line.startswith("#"):
 544                 result += line + "\n"
 545                 continue
 546
 547             if inDescriptionSection:
 548                 if line.startswith("Files:"):
 549                     inDescriptionSection = False
 550                 else:
 551                     continue
 552             else:
 553                 if line.startswith("Description:"):
 554                     inDescriptionSection = True
 555                     line += "\n"
 556                     for messageLine in message.split("\n"):
 557                         line += "\t" + messageLine + "\n"
 558
 559             result += line + "\n"
 560
 561         return result
 562
 563     def prepareSubmitTemplate(self):
 564         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 565         template = ""
 566         inFilesSection = False
 567         for line in p4_read_pipe_lines("change -o"):
 568             if line.endswith("\r\n"):
 569                 line = line[:-2] + "\n"
 570             if inFilesSection:
 571                 if line.startswith("\t"):
 572                     # path starts and ends with a tab
 573                     path = line[1:]
 574                     lastTab = path.rfind("\t")
 575                     if lastTab != -1:
 576                         path = path[:lastTab]
 577                         if not path.startswith(self.depotPath):
 578                             continue
 579                 else:
 580                     inFilesSection = False
 581             else:
 582                 if line.startswith("Files:"):
 583                     inFilesSection = True
 584
 585             template += line
 586
 587         return template
 588
 589     def applyCommit(self, id):
 590         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 591         diffOpts = ("", "-M")[self.detectRename]
 592         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 593         filesToAdd = set()
 594         filesToDelete = set()
 595         editedFiles = set()
 596         filesToChangeExecBit = {}
 597         for line in diff:
 598             diff = parseDiffTreeEntry(line)
 599             modifier = diff['status']
 600             path = diff['src']
 601             if modifier == "M":
 602                 p4_system("edit \"%s\"" % path)
 603                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 604                     filesToChangeExecBit[path] = diff['dst_mode']
 605                 editedFiles.add(path)
 606             elif modifier == "A":
 607                 filesToAdd.add(path)
 608                 filesToChangeExecBit[path] = diff['dst_mode']
 609                 if path in filesToDelete:
 610                     filesToDelete.remove(path)
 611             elif modifier == "D":
 612                 filesToDelete.add(path)
 613                 if path in filesToAdd:
 614                     filesToAdd.remove(path)
 615             elif modifier == "R":
 616                 src, dest = diff['src'], diff['dst']
 617                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 618                 p4_system("edit \"%s\"" % (dest))
 619                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 620                     filesToChangeExecBit[dest] = diff['dst_mode']
 621                 os.unlink(dest)
 622                 editedFiles.add(dest)
 623                 filesToDelete.add(src)
 624             else:
 625                 die("unknown modifier %s for %s" % (modifier, path))
 626
 627         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 628         patchcmd = diffcmd + " | git apply "
 629         tryPatchCmd = patchcmd + "--check -"
 630         applyPatchCmd = patchcmd + "--check --apply -"
 631
 632         if os.system(tryPatchCmd) != 0:
 633             print "Unfortunately applying the change failed!"
 634             print "What do you want to do?"
 635             response = "x"
 636             while response != "s" and response != "a" and response != "w":
 637                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 638                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 639             if response == "s":
 640                 print "Skipping! Good luck with the next patches..."
 641                 for f in editedFiles:
 642                     p4_system("revert \"%s\"" % f);
 643                 for f in filesToAdd:
 644                     system("rm %s" %f)
 645                 return
 646             elif response == "a":
 647                 os.system(applyPatchCmd)
 648                 if len(filesToAdd) > 0:
 649                     print "You may also want to call p4 add on the following files:"
 650                     print " ".join(filesToAdd)
 651                 if len(filesToDelete):
 652                     print "The following files should be scheduled for deletion with p4 delete:"
 653                     print " ".join(filesToDelete)
 654                 die("Please resolve and submit the conflict manually and "
 655                     + "continue afterwards with git-p4 submit --continue")
 656             elif response == "w":
 657                 system(diffcmd + " > patch.txt")
 658                 print "Patch saved to patch.txt in %s !" % self.clientPath
 659                 die("Please resolve and submit the conflict manually and "
 660                     "continue afterwards with git-p4 submit --continue")
 661
 662         system(applyPatchCmd)
 663
 664         for f in filesToAdd:
 665             p4_system("add \"%s\"" % f)
 666         for f in filesToDelete:
 667             p4_system("revert \"%s\"" % f)
 668             p4_system("delete \"%s\"" % f)
 669
 670         # Set/clear executable bits
 671         for f in filesToChangeExecBit.keys():
 672             mode = filesToChangeExecBit[f]
 673             setP4ExecBit(f, mode)
 674
 675         logMessage = extractLogMessageFromGitCommit(id)
 676         logMessage = logMessage.strip()
 677
 678         template = self.prepareSubmitTemplate()
 679
 680         if self.interactive:
 681             submitTemplate = self.prepareLogMessage(template, logMessage)
 682             if os.environ.has_key("P4DIFF"):
 683                 del(os.environ["P4DIFF"])
 684             diff = read_pipe("p4 diff -du ...")
 685
 686             newdiff = ""
 687             for newFile in filesToAdd:
 688                 newdiff += "==== new file ====\n"
 689                 newdiff += "--- /dev/null\n"
 690                 newdiff += "+++ %s\n" % newFile
 691                 f = open(newFile, "r")
 692                 for line in f.readlines():
 693                     newdiff += "+" + line
 694                 f.close()
 695
 696             separatorLine = "######## everything below this line is just the diff #######\n"
 697
 698             [handle, fileName] = tempfile.mkstemp()
 699             tmpFile = os.fdopen(handle, "w+")
 700             if self.isWindows:
 701                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 702                 separatorLine = separatorLine.replace("\n", "\r\n")
 703                 newdiff = newdiff.replace("\n", "\r\n")
 704             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 705             tmpFile.close()
 706             defaultEditor = "vi"
 707             if platform.system() == "Windows":
 708                 defaultEditor = "notepad"
 709             if os.environ.has_key("P4EDITOR"):
 710                 editor = os.environ.get("P4EDITOR")
 711             else:
 712                 editor = os.environ.get("EDITOR", defaultEditor);
 713             system(editor + " " + fileName)
 714             tmpFile = open(fileName, "rb")
 715             message = tmpFile.read()
 716             tmpFile.close()
 717             os.remove(fileName)
 718             submitTemplate = message[:message.index(separatorLine)]
 719             if self.isWindows:
 720                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 721
 722             write_pipe("p4 submit -i", submitTemplate)
 723         else:
 724             fileName = "submit.txt"
 725             file = open(fileName, "w+")
 726             file.write(self.prepareLogMessage(template, logMessage))
 727             file.close()
 728             print ("Perforce submit template written as %s. "
 729                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 730                    % (fileName, fileName))
 731
 732     def run(self, args):
 733         if len(args) == 0:
 734             self.master = currentGitBranch()
 735             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 736                 die("Detecting current git branch failed!")
 737         elif len(args) == 1:
 738             self.master = args[0]
 739         else:
 740             return False
 741
 742         allowSubmit = gitConfig("git-p4.allowSubmit")
 743         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 744             die("%s is not in git-p4.allowSubmit" % self.master)
 745
 746         [upstream, settings] = findUpstreamBranchPoint()
 747         self.depotPath = settings['depot-paths'][0]
 748         if len(self.origin) == 0:
 749             self.origin = upstream
 750
 751         if self.verbose:
 752             print "Origin branch is " + self.origin
 753
 754         if len(self.depotPath) == 0:
 755             print "Internal error: cannot locate perforce depot path from existing branches"
 756             sys.exit(128)
 757
 758         self.clientPath = p4Where(self.depotPath)
 759
 760         if len(self.clientPath) == 0:
 761             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 762             sys.exit(128)
 763
 764         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 765         self.oldWorkingDirectory = os.getcwd()
 766
 767         os.chdir(self.clientPath)
 768         print "Syncronizing p4 checkout..."
 769         p4_system("sync ...")
 770
 771         self.check()
 772
 773         commits = []
 774         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 775             commits.append(line.strip())
 776         commits.reverse()
 777
 778         while len(commits) > 0:
 779             commit = commits[0]
 780             commits = commits[1:]
 781             self.applyCommit(commit)
 782             if not self.interactive:
 783                 break
 784
 785         if len(commits) == 0:
 786             print "All changes applied!"
 787             os.chdir(self.oldWorkingDirectory)
 788
 789             sync = P4Sync()
 790             sync.run([])
 791
 792             rebase = P4Rebase()
 793             rebase.rebase()
 794
 795         return True
 796
 797 class P4Sync(Command):
 798     def __init__(self):
 799         Command.__init__(self)
 800         self.options = [
 801                 optparse.make_option("--branch", dest="branch"),
 802                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 803                 optparse.make_option("--changesfile", dest="changesFile"),
 804                 optparse.make_option("--silent", dest="silent", action="store_true"),
 805                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 806                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 807                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 808                                      help="Import into refs/heads/ , not refs/remotes"),
 809                 optparse.make_option("--max-changes", dest="maxChanges"),
 810                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 811                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 812                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 813                                      help="Only sync files that are included in the Perforce Client Spec")
 814         ]
 815         self.description = """Imports from Perforce into a git repository.\n
 816     example:
 817     //depot/my/project/ -- to import the current head
 818     //depot/my/project/@all -- to import everything
 819     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 820
 821     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 822
 823         self.usage += " //depot/path[@revRange]"
 824         self.silent = False
 825         self.createdBranches = Set()
 826         self.committedChanges = Set()
 827         self.branch = ""
 828         self.detectBranches = False
 829         self.detectLabels = False
 830         self.changesFile = ""
 831         self.syncWithOrigin = True
 832         self.verbose = False
 833         self.importIntoRemotes = True
 834         self.maxChanges = ""
 835         self.isWindows = (platform.system() == "Windows")
 836         self.keepRepoPath = False
 837         self.depotPaths = None
 838         self.p4BranchesInGit = []
 839         self.cloneExclude = []
 840         self.useClientSpec = False
 841         self.clientSpecDirs = []
 842
 843         if gitConfig("git-p4.syncFromOrigin") == "false":
 844             self.syncWithOrigin = False
 845
 846     def extractFilesFromCommit(self, commit):
 847         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 848                              for path in self.cloneExclude]
 849         files = []
 850         fnum = 0
 851         while commit.has_key("depotFile%s" % fnum):
 852             path =  commit["depotFile%s" % fnum]
 853
 854             if [p for p in self.cloneExclude
 855                 if path.startswith (p)]:
 856                 found = False
 857             else:
 858                 found = [p for p in self.depotPaths
 859                          if path.startswith (p)]
 860             if not found:
 861                 fnum = fnum + 1
 862                 continue
 863
 864             file = {}
 865             file["path"] = path
 866             file["rev"] = commit["rev%s" % fnum]
 867             file["action"] = commit["action%s" % fnum]
 868             file["type"] = commit["type%s" % fnum]
 869             files.append(file)
 870             fnum = fnum + 1
 871         return files
 872
 873     def stripRepoPath(self, path, prefixes):
 874         if self.keepRepoPath:
 875             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 876
 877         for p in prefixes:
 878             if path.startswith(p):
 879                 path = path[len(p):]
 880
 881         return path
 882
 883     def splitFilesIntoBranches(self, commit):
 884         branches = {}
 885         fnum = 0
 886         while commit.has_key("depotFile%s" % fnum):
 887             path =  commit["depotFile%s" % fnum]
 888             found = [p for p in self.depotPaths
 889                      if path.startswith (p)]
 890             if not found:
 891                 fnum = fnum + 1
 892                 continue
 893
 894             file = {}
 895             file["path"] = path
 896             file["rev"] = commit["rev%s" % fnum]
 897             file["action"] = commit["action%s" % fnum]
 898             file["type"] = commit["type%s" % fnum]
 899             fnum = fnum + 1
 900
 901             relPath = self.stripRepoPath(path, self.depotPaths)
 902
 903             for branch in self.knownBranches.keys():
 904
 905                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 906                 if relPath.startswith(branch + "/"):
 907                     if branch not in branches:
 908                         branches[branch] = []
 909                     branches[branch].append(file)
 910                     break
 911
 912         return branches
 913
 914     ## Should move this out, doesn't use SELF.
 915     def readP4Files(self, files):
 916         filesForCommit = []
 917         filesToRead = []
 918
 919         for f in files:
 920             includeFile = True
 921             for val in self.clientSpecDirs:
 922                 if f['path'].startswith(val[0]):
 923                     if val[1] <= 0:
 924                         includeFile = False
 925                     break
 926
 927             if includeFile:
 928                 filesForCommit.append(f)
 929                 if f['action'] != 'delete':
 930                     filesToRead.append(f)
 931
 932         filedata = []
 933         if len(filesToRead) > 0:
 934             filedata = p4CmdList('-x - print',
 935                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 936                                                   for f in filesToRead]),
 937                                  stdin_mode='w+')
 938
 939             if "p4ExitCode" in filedata[0]:
 940                 die("Problems executing p4. Error: [%d]."
 941                     % (filedata[0]['p4ExitCode']));
 942
 943         j = 0;
 944         contents = {}
 945         while j < len(filedata):
 946             stat = filedata[j]
 947             j += 1
 948             text = [];
 949             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 950                 text.append(filedata[j]['data'])
 951                 j += 1
 952             text = ''.join(text)
 953
 954             if not stat.has_key('depotFile'):
 955                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 956                 continue
 957
 958             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 959                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 960             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 961                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 962
 963             contents[stat['depotFile']] = text
 964
 965         for f in filesForCommit:
 966             path = f['path']
 967             if contents.has_key(path):
 968                 f['data'] = contents[path]
 969
 970         return filesForCommit
 971
 972     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 973         epoch = details["time"]
 974         author = details["user"]
 975
 976         if self.verbose:
 977             print "commit into %s" % branch
 978
 979         # start with reading files; if that fails, we should not
 980         # create a commit.
 981         new_files = []
 982         for f in files:
 983             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 984                 new_files.append (f)
 985             else:
 986                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 987         files = self.readP4Files(new_files)
 988
 989         self.gitStream.write("commit %s\n" % branch)
 990 #        gitStream.write("mark :%s\n" % details["change"])
 991         self.committedChanges.add(int(details["change"]))
 992         committer = ""
 993         if author not in self.users:
 994             self.getUserMapFromPerforceServer()
 995         if author in self.users:
 996             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 997         else:
 998             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 999
1000         self.gitStream.write("committer %s\n" % committer)
1001
1002         self.gitStream.write("data <<EOT\n")
1003         self.gitStream.write(details["desc"])
1004         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1005                              % (','.join (branchPrefixes), details["change"]))
1006         if len(details['options']) > 0:
1007             self.gitStream.write(": options = %s" % details['options'])
1008         self.gitStream.write("]\nEOT\n\n")
1009
1010         if len(parent) > 0:
1011             if self.verbose:
1012                 print "parent %s" % parent
1013             self.gitStream.write("from %s\n" % parent)
1014
1015         for file in files:
1016             if file["type"] == "apple":
1017                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1018                 continue
1019
1020             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1021             if file["action"] == "delete":
1022                 self.gitStream.write("D %s\n" % relPath)
1023             else:
1024                 data = file['data']
1025
1026                 mode = "644"
1027                 if isP4Exec(file["type"]):
1028                     mode = "755"
1029                 elif file["type"] == "symlink":
1030                     mode = "120000"
1031                     # p4 print on a symlink contains "target\n", so strip it off
1032                     data = data[:-1]
1033
1034                 if self.isWindows and file["type"].endswith("text"):
1035                     data = data.replace("\r\n", "\n")
1036
1037                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1038                 self.gitStream.write("data %s\n" % len(data))
1039                 self.gitStream.write(data)
1040                 self.gitStream.write("\n")
1041
1042         self.gitStream.write("\n")
1043
1044         change = int(details["change"])
1045
1046         if self.labels.has_key(change):
1047             label = self.labels[change]
1048             labelDetails = label[0]
1049             labelRevisions = label[1]
1050             if self.verbose:
1051                 print "Change %s is labelled %s" % (change, labelDetails)
1052
1053             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1054                                                     for p in branchPrefixes]))
1055
1056             if len(files) == len(labelRevisions):
1057
1058                 cleanedFiles = {}
1059                 for info in files:
1060                     if info["action"] == "delete":
1061                         continue
1062                     cleanedFiles[info["depotFile"]] = info["rev"]
1063
1064                 if cleanedFiles == labelRevisions:
1065                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1066                     self.gitStream.write("from %s\n" % branch)
1067
1068                     owner = labelDetails["Owner"]
1069                     tagger = ""
1070                     if author in self.users:
1071                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1072                     else:
1073                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1074                     self.gitStream.write("tagger %s\n" % tagger)
1075                     self.gitStream.write("data <<EOT\n")
1076                     self.gitStream.write(labelDetails["Description"])
1077                     self.gitStream.write("EOT\n\n")
1078
1079                 else:
1080                     if not self.silent:
1081                         print ("Tag %s does not match with change %s: files do not match."
1082                                % (labelDetails["label"], change))
1083
1084             else:
1085                 if not self.silent:
1086                     print ("Tag %s does not match with change %s: file count is different."
1087                            % (labelDetails["label"], change))
1088
1089     def getUserCacheFilename(self):
1090         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1091         return home + "/.gitp4-usercache.txt"
1092
1093     def getUserMapFromPerforceServer(self):
1094         if self.userMapFromPerforceServer:
1095             return
1096         self.users = {}
1097
1098         for output in p4CmdList("users"):
1099             if not output.has_key("User"):
1100                 continue
1101             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1102
1103
1104         s = ''
1105         for (key, val) in self.users.items():
1106             s += "%s\t%s\n" % (key, val)
1107
1108         open(self.getUserCacheFilename(), "wb").write(s)
1109         self.userMapFromPerforceServer = True
1110
1111     def loadUserMapFromCache(self):
1112         self.users = {}
1113         self.userMapFromPerforceServer = False
1114         try:
1115             cache = open(self.getUserCacheFilename(), "rb")
1116             lines = cache.readlines()
1117             cache.close()
1118             for line in lines:
1119                 entry = line.strip().split("\t")
1120                 self.users[entry[0]] = entry[1]
1121         except IOError:
1122             self.getUserMapFromPerforceServer()
1123
1124     def getLabels(self):
1125         self.labels = {}
1126
1127         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1128         if len(l) > 0 and not self.silent:
1129             print "Finding files belonging to labels in %s" % `self.depotPaths`
1130
1131         for output in l:
1132             label = output["label"]
1133             revisions = {}
1134             newestChange = 0
1135             if self.verbose:
1136                 print "Querying files for label %s" % label
1137             for file in p4CmdList("files "
1138                                   +  ' '.join (["%s...@%s" % (p, label)
1139                                                 for p in self.depotPaths])):
1140                 revisions[file["depotFile"]] = file["rev"]
1141                 change = int(file["change"])
1142                 if change > newestChange:
1143                     newestChange = change
1144
1145             self.labels[newestChange] = [output, revisions]
1146
1147         if self.verbose:
1148             print "Label changes: %s" % self.labels.keys()
1149
1150     def guessProjectName(self):
1151         for p in self.depotPaths:
1152             if p.endswith("/"):
1153                 p = p[:-1]
1154             p = p[p.strip().rfind("/") + 1:]
1155             if not p.endswith("/"):
1156                p += "/"
1157             return p
1158
1159     def getBranchMapping(self):
1160         lostAndFoundBranches = set()
1161
1162         for info in p4CmdList("branches"):
1163             details = p4Cmd("branch -o %s" % info["branch"])
1164             viewIdx = 0
1165             while details.has_key("View%s" % viewIdx):
1166                 paths = details["View%s" % viewIdx].split(" ")
1167                 viewIdx = viewIdx + 1
1168                 # require standard //depot/foo/... //depot/bar/... mapping
1169                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1170                     continue
1171                 source = paths[0]
1172                 destination = paths[1]
1173                 ## HACK
1174                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1175                     source = source[len(self.depotPaths[0]):-4]
1176                     destination = destination[len(self.depotPaths[0]):-4]
1177
1178                     if destination in self.knownBranches:
1179                         if not self.silent:
1180                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1181                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1182                         continue
1183
1184                     self.knownBranches[destination] = source
1185
1186                     lostAndFoundBranches.discard(destination)
1187
1188                     if source not in self.knownBranches:
1189                         lostAndFoundBranches.add(source)
1190
1191
1192         for branch in lostAndFoundBranches:
1193             self.knownBranches[branch] = branch
1194
1195     def getBranchMappingFromGitBranches(self):
1196         branches = p4BranchesInGit(self.importIntoRemotes)
1197         for branch in branches.keys():
1198             if branch == "master":
1199                 branch = "main"
1200             else:
1201                 branch = branch[len(self.projectName):]
1202             self.knownBranches[branch] = branch
1203
1204     def listExistingP4GitBranches(self):
1205         # branches holds mapping from name to commit
1206         branches = p4BranchesInGit(self.importIntoRemotes)
1207         self.p4BranchesInGit = branches.keys()
1208         for branch in branches.keys():
1209             self.initialParents[self.refPrefix + branch] = branches[branch]
1210
1211     def updateOptionDict(self, d):
1212         option_keys = {}
1213         if self.keepRepoPath:
1214             option_keys['keepRepoPath'] = 1
1215
1216         d["options"] = ' '.join(sorted(option_keys.keys()))
1217
1218     def readOptions(self, d):
1219         self.keepRepoPath = (d.has_key('options')
1220                              and ('keepRepoPath' in d['options']))
1221
1222     def gitRefForBranch(self, branch):
1223         if branch == "main":
1224             return self.refPrefix + "master"
1225
1226         if len(branch) <= 0:
1227             return branch
1228
1229         return self.refPrefix + self.projectName + branch
1230
1231     def gitCommitByP4Change(self, ref, change):
1232         if self.verbose:
1233             print "looking in ref " + ref + " for change %s using bisect..." % change
1234
1235         earliestCommit = ""
1236         latestCommit = parseRevision(ref)
1237
1238         while True:
1239             if self.verbose:
1240                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1241             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1242             if len(next) == 0:
1243                 if self.verbose:
1244                     print "argh"
1245                 return ""
1246             log = extractLogMessageFromGitCommit(next)
1247             settings = extractSettingsGitLog(log)
1248             currentChange = int(settings['change'])
1249             if self.verbose:
1250                 print "current change %s" % currentChange
1251
1252             if currentChange == change:
1253                 if self.verbose:
1254                     print "found %s" % next
1255                 return next
1256
1257             if currentChange < change:
1258                 earliestCommit = "^%s" % next
1259             else:
1260                 latestCommit = "%s" % next
1261
1262         return ""
1263
1264     def importNewBranch(self, branch, maxChange):
1265         # make fast-import flush all changes to disk and update the refs using the checkpoint
1266         # command so that we can try to find the branch parent in the git history
1267         self.gitStream.write("checkpoint\n\n");
1268         self.gitStream.flush();
1269         branchPrefix = self.depotPaths[0] + branch + "/"
1270         range = "@1,%s" % maxChange
1271         #print "prefix" + branchPrefix
1272         changes = p4ChangesForPaths([branchPrefix], range)
1273         if len(changes) <= 0:
1274             return False
1275         firstChange = changes[0]
1276         #print "first change in branch: %s" % firstChange
1277         sourceBranch = self.knownBranches[branch]
1278         sourceDepotPath = self.depotPaths[0] + sourceBranch
1279         sourceRef = self.gitRefForBranch(sourceBranch)
1280         #print "source " + sourceBranch
1281
1282         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1283         #print "branch parent: %s" % branchParentChange
1284         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1285         if len(gitParent) > 0:
1286             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1287             #print "parent git commit: %s" % gitParent
1288
1289         self.importChanges(changes)
1290         return True
1291
1292     def importChanges(self, changes):
1293         cnt = 1
1294         for change in changes:
1295             description = p4Cmd("describe %s" % change)
1296             self.updateOptionDict(description)
1297
1298             if not self.silent:
1299                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1300                 sys.stdout.flush()
1301             cnt = cnt + 1
1302
1303             try:
1304                 if self.detectBranches:
1305                     branches = self.splitFilesIntoBranches(description)
1306                     for branch in branches.keys():
1307                         ## HACK  --hwn
1308                         branchPrefix = self.depotPaths[0] + branch + "/"
1309
1310                         parent = ""
1311
1312                         filesForCommit = branches[branch]
1313
1314                         if self.verbose:
1315                             print "branch is %s" % branch
1316
1317                         self.updatedBranches.add(branch)
1318
1319                         if branch not in self.createdBranches:
1320                             self.createdBranches.add(branch)
1321                             parent = self.knownBranches[branch]
1322                             if parent == branch:
1323                                 parent = ""
1324                             else:
1325                                 fullBranch = self.projectName + branch
1326                                 if fullBranch not in self.p4BranchesInGit:
1327                                     if not self.silent:
1328                                         print("\n    Importing new branch %s" % fullBranch);
1329                                     if self.importNewBranch(branch, change - 1):
1330                                         parent = ""
1331                                         self.p4BranchesInGit.append(fullBranch)
1332                                     if not self.silent:
1333                                         print("\n    Resuming with change %s" % change);
1334
1335                                 if self.verbose:
1336                                     print "parent determined through known branches: %s" % parent
1337
1338                         branch = self.gitRefForBranch(branch)
1339                         parent = self.gitRefForBranch(parent)
1340
1341                         if self.verbose:
1342                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1343
1344                         if len(parent) == 0 and branch in self.initialParents:
1345                             parent = self.initialParents[branch]
1346                             del self.initialParents[branch]
1347
1348                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1349                 else:
1350                     files = self.extractFilesFromCommit(description)
1351                     self.commit(description, files, self.branch, self.depotPaths,
1352                                 self.initialParent)
1353                     self.initialParent = ""
1354             except IOError:
1355                 print self.gitError.read()
1356                 sys.exit(1)
1357
1358     def importHeadRevision(self, revision):
1359         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1360
1361         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1362         details["desc"] = ("Initial import of %s from the state at revision %s"
1363                            % (' '.join(self.depotPaths), revision))
1364         details["change"] = revision
1365         newestRevision = 0
1366
1367         fileCnt = 0
1368         for info in p4CmdList("files "
1369                               +  ' '.join(["%s...%s"
1370                                            % (p, revision)
1371                                            for p in self.depotPaths])):
1372
1373             if info['code'] == 'error':
1374                 sys.stderr.write("p4 returned an error: %s\n"
1375                                  % info['data'])
1376                 sys.exit(1)
1377
1378
1379             change = int(info["change"])
1380             if change > newestRevision:
1381                 newestRevision = change
1382
1383             if info["action"] == "delete":
1384                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1385                 #fileCnt = fileCnt + 1
1386                 continue
1387
1388             for prop in ["depotFile", "rev", "action", "type" ]:
1389                 details["%s%s" % (prop, fileCnt)] = info[prop]
1390
1391             fileCnt = fileCnt + 1
1392
1393         details["change"] = newestRevision
1394         self.updateOptionDict(details)
1395         try:
1396             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1397         except IOError:
1398             print "IO error with git fast-import. Is your git version recent enough?"
1399             print self.gitError.read()
1400
1401
1402     def getClientSpec(self):
1403         specList = p4CmdList( "client -o" )
1404         temp = {}
1405         for entry in specList:
1406             for k,v in entry.iteritems():
1407                 if k.startswith("View"):
1408                     if v.startswith('"'):
1409                         start = 1
1410                     else:
1411                         start = 0
1412                     index = v.find("...")
1413                     v = v[start:index]
1414                     if v.startswith("-"):
1415                         v = v[1:]
1416                         temp[v] = -len(v)
1417                     else:
1418                         temp[v] = len(v)
1419         self.clientSpecDirs = temp.items()
1420         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1421
1422     def run(self, args):
1423         self.depotPaths = []
1424         self.changeRange = ""
1425         self.initialParent = ""
1426         self.previousDepotPaths = []
1427
1428         # map from branch depot path to parent branch
1429         self.knownBranches = {}
1430         self.initialParents = {}
1431         self.hasOrigin = originP4BranchesExist()
1432         if not self.syncWithOrigin:
1433             self.hasOrigin = False
1434
1435         if self.importIntoRemotes:
1436             self.refPrefix = "refs/remotes/p4/"
1437         else:
1438             self.refPrefix = "refs/heads/p4/"
1439
1440         if self.syncWithOrigin and self.hasOrigin:
1441             if not self.silent:
1442                 print "Syncing with origin first by calling git fetch origin"
1443             system("git fetch origin")
1444
1445         if len(self.branch) == 0:
1446             self.branch = self.refPrefix + "master"
1447             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1448                 system("git update-ref %s refs/heads/p4" % self.branch)
1449                 system("git branch -D p4");
1450             # create it /after/ importing, when master exists
1451             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1452                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1453
1454         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1455             self.getClientSpec()
1456
1457         # TODO: should always look at previous commits,
1458         # merge with previous imports, if possible.
1459         if args == []:
1460             if self.hasOrigin:
1461                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1462             self.listExistingP4GitBranches()
1463
1464             if len(self.p4BranchesInGit) > 1:
1465                 if not self.silent:
1466                     print "Importing from/into multiple branches"
1467                 self.detectBranches = True
1468
1469             if self.verbose:
1470                 print "branches: %s" % self.p4BranchesInGit
1471
1472             p4Change = 0
1473             for branch in self.p4BranchesInGit:
1474                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1475
1476                 settings = extractSettingsGitLog(logMsg)
1477
1478                 self.readOptions(settings)
1479                 if (settings.has_key('depot-paths')
1480                     and settings.has_key ('change')):
1481                     change = int(settings['change']) + 1
1482                     p4Change = max(p4Change, change)
1483
1484                     depotPaths = sorted(settings['depot-paths'])
1485                     if self.previousDepotPaths == []:
1486                         self.previousDepotPaths = depotPaths
1487                     else:
1488                         paths = []
1489                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1490                             for i in range(0, min(len(cur), len(prev))):
1491                                 if cur[i] <> prev[i]:
1492                                     i = i - 1
1493                                     break
1494
1495                             paths.append (cur[:i + 1])
1496
1497                         self.previousDepotPaths = paths
1498
1499             if p4Change > 0:
1500                 self.depotPaths = sorted(self.previousDepotPaths)
1501                 self.changeRange = "@%s,#head" % p4Change
1502                 if not self.detectBranches:
1503                     self.initialParent = parseRevision(self.branch)
1504                 if not self.silent and not self.detectBranches:
1505                     print "Performing incremental import into %s git branch" % self.branch
1506
1507         if not self.branch.startswith("refs/"):
1508             self.branch = "refs/heads/" + self.branch
1509
1510         if len(args) == 0 and self.depotPaths:
1511             if not self.silent:
1512                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1513         else:
1514             if self.depotPaths and self.depotPaths != args:
1515                 print ("previous import used depot path %s and now %s was specified. "
1516                        "This doesn't work!" % (' '.join (self.depotPaths),
1517                                                ' '.join (args)))
1518                 sys.exit(1)
1519
1520             self.depotPaths = sorted(args)
1521
1522         revision = ""
1523         self.users = {}
1524
1525         newPaths = []
1526         for p in self.depotPaths:
1527             if p.find("@") != -1:
1528                 atIdx = p.index("@")
1529                 self.changeRange = p[atIdx:]
1530                 if self.changeRange == "@all":
1531                     self.changeRange = ""
1532                 elif ',' not in self.changeRange:
1533                     revision = self.changeRange
1534                     self.changeRange = ""
1535                 p = p[:atIdx]
1536             elif p.find("#") != -1:
1537                 hashIdx = p.index("#")
1538                 revision = p[hashIdx:]
1539                 p = p[:hashIdx]
1540             elif self.previousDepotPaths == []:
1541                 revision = "#head"
1542
1543             p = re.sub ("\.\.\.$", "", p)
1544             if not p.endswith("/"):
1545                 p += "/"
1546
1547             newPaths.append(p)
1548
1549         self.depotPaths = newPaths
1550
1551
1552         self.loadUserMapFromCache()
1553         self.labels = {}
1554         if self.detectLabels:
1555             self.getLabels();
1556
1557         if self.detectBranches:
1558             ## FIXME - what's a P4 projectName ?
1559             self.projectName = self.guessProjectName()
1560
1561             if self.hasOrigin:
1562                 self.getBranchMappingFromGitBranches()
1563             else:
1564                 self.getBranchMapping()
1565             if self.verbose:
1566                 print "p4-git branches: %s" % self.p4BranchesInGit
1567                 print "initial parents: %s" % self.initialParents
1568             for b in self.p4BranchesInGit:
1569                 if b != "master":
1570
1571                     ## FIXME
1572                     b = b[len(self.projectName):]
1573                 self.createdBranches.add(b)
1574
1575         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1576
1577         importProcess = subprocess.Popen(["git", "fast-import"],
1578                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1579                                          stderr=subprocess.PIPE);
1580         self.gitOutput = importProcess.stdout
1581         self.gitStream = importProcess.stdin
1582         self.gitError = importProcess.stderr
1583
1584         if revision:
1585             self.importHeadRevision(revision)
1586         else:
1587             changes = []
1588
1589             if len(self.changesFile) > 0:
1590                 output = open(self.changesFile).readlines()
1591                 changeSet = Set()
1592                 for line in output:
1593                     changeSet.add(int(line))
1594
1595                 for change in changeSet:
1596                     changes.append(change)
1597
1598                 changes.sort()
1599             else:
1600                 if self.verbose:
1601                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1602                                                               self.changeRange)
1603                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1604
1605                 if len(self.maxChanges) > 0:
1606                     changes = changes[:min(int(self.maxChanges), len(changes))]
1607
1608             if len(changes) == 0:
1609                 if not self.silent:
1610                     print "No changes to import!"
1611                 return True
1612
1613             if not self.silent and not self.detectBranches:
1614                 print "Import destination: %s" % self.branch
1615
1616             self.updatedBranches = set()
1617
1618             self.importChanges(changes)
1619
1620             if not self.silent:
1621                 print ""
1622                 if len(self.updatedBranches) > 0:
1623                     sys.stdout.write("Updated branches: ")
1624                     for b in self.updatedBranches:
1625                         sys.stdout.write("%s " % b)
1626                     sys.stdout.write("\n")
1627
1628         self.gitStream.close()
1629         if importProcess.wait() != 0:
1630             die("fast-import failed: %s" % self.gitError.read())
1631         self.gitOutput.close()
1632         self.gitError.close()
1633
1634         return True
1635
1636 class P4Rebase(Command):
1637     def __init__(self):
1638         Command.__init__(self)
1639         self.options = [ ]
1640         self.description = ("Fetches the latest revision from perforce and "
1641                             + "rebases the current work (branch) against it")
1642         self.verbose = False
1643
1644     def run(self, args):
1645         sync = P4Sync()
1646         sync.run([])
1647
1648         return self.rebase()
1649
1650     def rebase(self):
1651         if os.system("git update-index --refresh") != 0:
1652             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1653         if len(read_pipe("git diff-index HEAD --")) > 0:
1654             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1655
1656         [upstream, settings] = findUpstreamBranchPoint()
1657         if len(upstream) == 0:
1658             die("Cannot find upstream branchpoint for rebase")
1659
1660         # the branchpoint may be p4/foo~3, so strip off the parent
1661         upstream = re.sub("~[0-9]+$", "", upstream)
1662
1663         print "Rebasing the current branch onto %s" % upstream
1664         oldHead = read_pipe("git rev-parse HEAD").strip()
1665         system("git rebase %s" % upstream)
1666         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1667         return True
1668
1669 class P4Clone(P4Sync):
1670     def __init__(self):
1671         P4Sync.__init__(self)
1672         self.description = "Creates a new git repository and imports from Perforce into it"
1673         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1674         self.options += [
1675             optparse.make_option("--destination", dest="cloneDestination",
1676                                  action='store', default=None,
1677                                  help="where to leave result of the clone"),
1678             optparse.make_option("-/", dest="cloneExclude",
1679                                  action="append", type="string",
1680                                  help="exclude depot path")
1681         ]
1682         self.cloneDestination = None
1683         self.needsGit = False
1684
1685     # This is required for the "append" cloneExclude action
1686     def ensure_value(self, attr, value):
1687         if not hasattr(self, attr) or getattr(self, attr) is None:
1688             setattr(self, attr, value)
1689         return getattr(self, attr)
1690
1691     def defaultDestination(self, args):
1692         ## TODO: use common prefix of args?
1693         depotPath = args[0]
1694         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1695         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1696         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1697         depotDir = re.sub(r"/$", "", depotDir)
1698         return os.path.split(depotDir)[1]
1699
1700     def run(self, args):
1701         if len(args) < 1:
1702             return False
1703
1704         if self.keepRepoPath and not self.cloneDestination:
1705             sys.stderr.write("Must specify destination for --keep-path\n")
1706             sys.exit(1)
1707
1708         depotPaths = args
1709
1710         if not self.cloneDestination and len(depotPaths) > 1:
1711             self.cloneDestination = depotPaths[-1]
1712             depotPaths = depotPaths[:-1]
1713
1714         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1715         for p in depotPaths:
1716             if not p.startswith("//"):
1717                 return False
1718
1719         if not self.cloneDestination:
1720             self.cloneDestination = self.defaultDestination(args)
1721
1722         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1723         if not os.path.exists(self.cloneDestination):
1724             os.makedirs(self.cloneDestination)
1725         os.chdir(self.cloneDestination)
1726         system("git init")
1727         self.gitdir = os.getcwd() + "/.git"
1728         if not P4Sync.run(self, depotPaths):
1729             return False
1730         if self.branch != "master":
1731             if gitBranchExists("refs/remotes/p4/master"):
1732                 system("git branch master refs/remotes/p4/master")
1733                 system("git checkout -f")
1734             else:
1735                 print "Could not detect main branch. No checkout/master branch created."
1736
1737         return True
1738
1739 class P4Branches(Command):
1740     def __init__(self):
1741         Command.__init__(self)
1742         self.options = [ ]
1743         self.description = ("Shows the git branches that hold imports and their "
1744                             + "corresponding perforce depot paths")
1745         self.verbose = False
1746
1747     def run(self, args):
1748         if originP4BranchesExist():
1749             createOrUpdateBranchesFromOrigin()
1750
1751         cmdline = "git rev-parse --symbolic "
1752         cmdline += " --remotes"
1753
1754         for line in read_pipe_lines(cmdline):
1755             line = line.strip()
1756
1757             if not line.startswith('p4/') or line == "p4/HEAD":
1758                 continue
1759             branch = line
1760
1761             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1762             settings = extractSettingsGitLog(log)
1763
1764             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1765         return True
1766
1767 class HelpFormatter(optparse.IndentedHelpFormatter):
1768     def __init__(self):
1769         optparse.IndentedHelpFormatter.__init__(self)
1770
1771     def format_description(self, description):
1772         if description:
1773             return description + "\n"
1774         else:
1775             return ""
1776
1777 def printUsage(commands):
1778     print "usage: %s <command> [options]" % sys.argv[0]
1779     print ""
1780     print "valid commands: %s" % ", ".join(commands)
1781     print ""
1782     print "Try %s <command> --help for command specific help." % sys.argv[0]
1783     print ""
1784
1785 commands = {
1786     "debug" : P4Debug,
1787     "submit" : P4Submit,
1788     "commit" : P4Submit,
1789     "sync" : P4Sync,
1790     "rebase" : P4Rebase,
1791     "clone" : P4Clone,
1792     "rollback" : P4RollBack,
1793     "branches" : P4Branches
1794 }
1795
1796
1797 def main():
1798     if len(sys.argv[1:]) == 0:
1799         printUsage(commands.keys())
1800         sys.exit(2)
1801
1802     cmd = ""
1803     cmdName = sys.argv[1]
1804     try:
1805         klass = commands[cmdName]
1806         cmd = klass()
1807     except KeyError:
1808         print "unknown command %s" % cmdName
1809         print ""
1810         printUsage(commands.keys())
1811         sys.exit(2)
1812
1813     options = cmd.options
1814     cmd.gitdir = os.environ.get("GIT_DIR", None)
1815
1816     args = sys.argv[2:]
1817
1818     if len(options) > 0:
1819         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1820
1821         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1822                                        options,
1823                                        description = cmd.description,
1824                                        formatter = HelpFormatter())
1825
1826         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1827     global verbose
1828     verbose = cmd.verbose
1829     if cmd.needsGit:
1830         if cmd.gitdir == None:
1831             cmd.gitdir = os.path.abspath(".git")
1832             if not isValidGitDir(cmd.gitdir):
1833                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1834                 if os.path.exists(cmd.gitdir):
1835                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1836                     if len(cdup) > 0:
1837                         os.chdir(cdup);
1838
1839         if not isValidGitDir(cmd.gitdir):
1840             if isValidGitDir(cmd.gitdir + "/.git"):
1841                 cmd.gitdir += "/.git"
1842             else:
1843                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1844
1845         os.environ["GIT_DIR"] = cmd.gitdir
1846
1847     if not cmd.run(args):
1848         parser.print_help()
1849
1850
1851 if __name__ == '__main__':
1852     main()