contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     if os.name == 'nt':
  56         os.environ['PWD']=dir
  57     os.chdir(dir)
  58
  59 def die(msg):
  60     if verbose:
  61         raise Exception(msg)
  62     else:
  63         sys.stderr.write(msg + "\n")
  64         sys.exit(1)
  65
  66 def write_pipe(c, str):
  67     if verbose:
  68         sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70     pipe = os.popen(c, 'w')
  71     val = pipe.write(str)
  72     if pipe.close():
  73         die('Command failed: %s' % c)
  74
  75     return val
  76
  77 def p4_write_pipe(c, str):
  78     real_cmd = p4_build_cmd(c)
  79     return write_pipe(real_cmd, str)
  80
  81 def read_pipe(c, ignore_error=False):
  82     if verbose:
  83         sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85     pipe = os.popen(c, 'rb')
  86     val = pipe.read()
  87     if pipe.close() and not ignore_error:
  88         die('Command failed: %s' % c)
  89
  90     return val
  91
  92 def p4_read_pipe(c, ignore_error=False):
  93     real_cmd = p4_build_cmd(c)
  94     return read_pipe(real_cmd, ignore_error)
  95
  96 def read_pipe_lines(c):
  97     if verbose:
  98         sys.stderr.write('Reading pipe: %s\n' % c)
  99     ## todo: check return status
 100     pipe = os.popen(c, 'rb')
 101     val = pipe.readlines()
 102     if pipe.close():
 103         die('Command failed: %s' % c)
 104
 105     return val
 106
 107 def p4_read_pipe_lines(c):
 108     """Specifically invoke p4 on the command supplied. """
 109     real_cmd = p4_build_cmd(c)
 110     return read_pipe_lines(real_cmd)
 111
 112 def system(cmd):
 113     if verbose:
 114         sys.stderr.write("executing %s\n" % cmd)
 115     if os.system(cmd) != 0:
 116         die("command failed: %s" % cmd)
 117
 118 def p4_system(cmd):
 119     """Specifically invoke p4 as the system command. """
 120     real_cmd = p4_build_cmd(cmd)
 121     return system(real_cmd)
 122
 123 def isP4Exec(kind):
 124     """Determine if a Perforce 'kind' should have execute permission
 125
 126     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128     a plus sign, it is also executable"""
 129     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131 def setP4ExecBit(file, mode):
 132     # Reopens an already open file and changes the execute bit to match
 133     # the execute bit setting in the passed in mode.
 134
 135     p4Type = "+x"
 136
 137     if not isModeExec(mode):
 138         p4Type = getP4OpenedType(file)
 139         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141         if p4Type[-1] == "+":
 142             p4Type = p4Type[0:-1]
 143
 144     p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146 def getP4OpenedType(file):
 147     # Returns the perforce file type for the given file.
 148
 149     result = p4_read_pipe("opened %s" % file)
 150     match = re.match(".*\((.+)\)\r?$", result)
 151     if match:
 152         return match.group(1)
 153     else:
 154         die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156 def diffTreePattern():
 157     # This is a simple generator for the diff tree regex pattern. This could be
 158     # a class variable if this and parseDiffTreeEntry were a part of a class.
 159     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160     while True:
 161         yield pattern
 162
 163 def parseDiffTreeEntry(entry):
 164     """Parses a single diff tree entry into its component elements.
 165
 166     See git-diff-tree(1) manpage for details about the format of the diff
 167     output. This method returns a dictionary with the following elements:
 168
 169     src_mode - The mode of the source file
 170     dst_mode - The mode of the destination file
 171     src_sha1 - The sha1 for the source file
 172     dst_sha1 - The sha1 fr the destination file
 173     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174     status_score - The score for the status (applicable for 'C' and 'R'
 175                    statuses). This is None if there is no score.
 176     src - The path for the source file.
 177     dst - The path for the destination file. This is only present for
 178           copy or renames. If it is not present, this is None.
 179
 180     If the pattern is not matched, None is returned."""
 181
 182     match = diffTreePattern().next().match(entry)
 183     if match:
 184         return {
 185             'src_mode': match.group(1),
 186             'dst_mode': match.group(2),
 187             'src_sha1': match.group(3),
 188             'dst_sha1': match.group(4),
 189             'status': match.group(5),
 190             'status_score': match.group(6),
 191             'src': match.group(7),
 192             'dst': match.group(10)
 193         }
 194     return None
 195
 196 def isModeExec(mode):
 197     # Returns True if the given git mode represents an executable file,
 198     # otherwise False.
 199     return mode[-3:] == "755"
 200
 201 def isModeExecChanged(src_mode, dst_mode):
 202     return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205     cmd = p4_build_cmd("-G %s" % (cmd))
 206     if verbose:
 207         sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209     # Use a temporary file to avoid deadlocks without
 210     # subprocess.communicate(), which would put another copy
 211     # of stdout into memory.
 212     stdin_file = None
 213     if stdin is not None:
 214         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215         stdin_file.write(stdin)
 216         stdin_file.flush()
 217         stdin_file.seek(0)
 218
 219     p4 = subprocess.Popen(cmd, shell=True,
 220                           stdin=stdin_file,
 221                           stdout=subprocess.PIPE)
 222
 223     result = []
 224     try:
 225         while True:
 226             entry = marshal.load(p4.stdout)
 227             result.append(entry)
 228     except EOFError:
 229         pass
 230     exitCode = p4.wait()
 231     if exitCode != 0:
 232         entry = {}
 233         entry["p4ExitCode"] = exitCode
 234         result.append(entry)
 235
 236     return result
 237
 238 def p4Cmd(cmd):
 239     list = p4CmdList(cmd)
 240     result = {}
 241     for entry in list:
 242         result.update(entry)
 243     return result;
 244
 245 def p4Where(depotPath):
 246     if not depotPath.endswith("/"):
 247         depotPath += "/"
 248     depotPath = depotPath + "..."
 249     outputList = p4CmdList("where %s" % depotPath)
 250     output = None
 251     for entry in outputList:
 252         if "depotFile" in entry:
 253             if entry["depotFile"] == depotPath:
 254                 output = entry
 255                 break
 256         elif "data" in entry:
 257             data = entry.get("data")
 258             space = data.find(" ")
 259             if data[:space] == depotPath:
 260                 output = entry
 261                 break
 262     if output == None:
 263         return ""
 264     if output["code"] == "error":
 265         return ""
 266     clientPath = ""
 267     if "path" in output:
 268         clientPath = output.get("path")
 269     elif "data" in output:
 270         data = output.get("data")
 271         lastSpace = data.rfind(" ")
 272         clientPath = data[lastSpace + 1:]
 273
 274     if clientPath.endswith("..."):
 275         clientPath = clientPath[:-3]
 276     return clientPath
 277
 278 def currentGitBranch():
 279     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 280
 281 def isValidGitDir(path):
 282     if (os.path.exists(path + "/HEAD")
 283         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 284         return True;
 285     return False
 286
 287 def parseRevision(ref):
 288     return read_pipe("git rev-parse %s" % ref).strip()
 289
 290 def extractLogMessageFromGitCommit(commit):
 291     logMessage = ""
 292
 293     ## fixme: title is first line of commit, not 1st paragraph.
 294     foundTitle = False
 295     for log in read_pipe_lines("git cat-file commit %s" % commit):
 296        if not foundTitle:
 297            if len(log) == 1:
 298                foundTitle = True
 299            continue
 300
 301        logMessage += log
 302     return logMessage
 303
 304 def extractSettingsGitLog(log):
 305     values = {}
 306     for line in log.split("\n"):
 307         line = line.strip()
 308         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 309         if not m:
 310             continue
 311
 312         assignments = m.group(1).split (':')
 313         for a in assignments:
 314             vals = a.split ('=')
 315             key = vals[0].strip()
 316             val = ('='.join (vals[1:])).strip()
 317             if val.endswith ('\"') and val.startswith('"'):
 318                 val = val[1:-1]
 319
 320             values[key] = val
 321
 322     paths = values.get("depot-paths")
 323     if not paths:
 324         paths = values.get("depot-path")
 325     if paths:
 326         values['depot-paths'] = paths.split(',')
 327     return values
 328
 329 def gitBranchExists(branch):
 330     proc = subprocess.Popen(["git", "rev-parse", branch],
 331                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 332     return proc.wait() == 0;
 333
 334 _gitConfig = {}
 335 def gitConfig(key):
 336     if not _gitConfig.has_key(key):
 337         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 338     return _gitConfig[key]
 339
 340 def p4BranchesInGit(branchesAreInRemotes = True):
 341     branches = {}
 342
 343     cmdline = "git rev-parse --symbolic "
 344     if branchesAreInRemotes:
 345         cmdline += " --remotes"
 346     else:
 347         cmdline += " --branches"
 348
 349     for line in read_pipe_lines(cmdline):
 350         line = line.strip()
 351
 352         ## only import to p4/
 353         if not line.startswith('p4/') or line == "p4/HEAD":
 354             continue
 355         branch = line
 356
 357         # strip off p4
 358         branch = re.sub ("^p4/", "", line)
 359
 360         branches[branch] = parseRevision(line)
 361     return branches
 362
 363 def findUpstreamBranchPoint(head = "HEAD"):
 364     branches = p4BranchesInGit()
 365     # map from depot-path to branch name
 366     branchByDepotPath = {}
 367     for branch in branches.keys():
 368         tip = branches[branch]
 369         log = extractLogMessageFromGitCommit(tip)
 370         settings = extractSettingsGitLog(log)
 371         if settings.has_key("depot-paths"):
 372             paths = ",".join(settings["depot-paths"])
 373             branchByDepotPath[paths] = "remotes/p4/" + branch
 374
 375     settings = None
 376     parent = 0
 377     while parent < 65535:
 378         commit = head + "~%s" % parent
 379         log = extractLogMessageFromGitCommit(commit)
 380         settings = extractSettingsGitLog(log)
 381         if settings.has_key("depot-paths"):
 382             paths = ",".join(settings["depot-paths"])
 383             if branchByDepotPath.has_key(paths):
 384                 return [branchByDepotPath[paths], settings]
 385
 386         parent = parent + 1
 387
 388     return ["", settings]
 389
 390 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 391     if not silent:
 392         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 393                % localRefPrefix)
 394
 395     originPrefix = "origin/p4/"
 396
 397     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 398         line = line.strip()
 399         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 400             continue
 401
 402         headName = line[len(originPrefix):]
 403         remoteHead = localRefPrefix + headName
 404         originHead = line
 405
 406         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 407         if (not original.has_key('depot-paths')
 408             or not original.has_key('change')):
 409             continue
 410
 411         update = False
 412         if not gitBranchExists(remoteHead):
 413             if verbose:
 414                 print "creating %s" % remoteHead
 415             update = True
 416         else:
 417             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 418             if settings.has_key('change') > 0:
 419                 if settings['depot-paths'] == original['depot-paths']:
 420                     originP4Change = int(original['change'])
 421                     p4Change = int(settings['change'])
 422                     if originP4Change > p4Change:
 423                         print ("%s (%s) is newer than %s (%s). "
 424                                "Updating p4 branch from origin."
 425                                % (originHead, originP4Change,
 426                                   remoteHead, p4Change))
 427                         update = True
 428                 else:
 429                     print ("Ignoring: %s was imported from %s while "
 430                            "%s was imported from %s"
 431                            % (originHead, ','.join(original['depot-paths']),
 432                               remoteHead, ','.join(settings['depot-paths'])))
 433
 434         if update:
 435             system("git update-ref %s %s" % (remoteHead, originHead))
 436
 437 def originP4BranchesExist():
 438         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 439
 440 def p4ChangesForPaths(depotPaths, changeRange):
 441     assert depotPaths
 442     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 443                                                         for p in depotPaths]))
 444
 445     changes = []
 446     for line in output:
 447         changeNum = line.split(" ")[1]
 448         changes.append(int(changeNum))
 449
 450     changes.sort()
 451     return changes
 452
 453 class Command:
 454     def __init__(self):
 455         self.usage = "usage: %prog [options]"
 456         self.needsGit = True
 457
 458 class P4Debug(Command):
 459     def __init__(self):
 460         Command.__init__(self)
 461         self.options = [
 462             optparse.make_option("--verbose", dest="verbose", action="store_true",
 463                                  default=False),
 464             ]
 465         self.description = "A tool to debug the output of p4 -G."
 466         self.needsGit = False
 467         self.verbose = False
 468
 469     def run(self, args):
 470         j = 0
 471         for output in p4CmdList(" ".join(args)):
 472             print 'Element: %d' % j
 473             j += 1
 474             print output
 475         return True
 476
 477 class P4RollBack(Command):
 478     def __init__(self):
 479         Command.__init__(self)
 480         self.options = [
 481             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 482             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 483         ]
 484         self.description = "A tool to debug the multi-branch import. Don't use :)"
 485         self.verbose = False
 486         self.rollbackLocalBranches = False
 487
 488     def run(self, args):
 489         if len(args) != 1:
 490             return False
 491         maxChange = int(args[0])
 492
 493         if "p4ExitCode" in p4Cmd("changes -m 1"):
 494             die("Problems executing p4");
 495
 496         if self.rollbackLocalBranches:
 497             refPrefix = "refs/heads/"
 498             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 499         else:
 500             refPrefix = "refs/remotes/"
 501             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 502
 503         for line in lines:
 504             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 505                 line = line.strip()
 506                 ref = refPrefix + line
 507                 log = extractLogMessageFromGitCommit(ref)
 508                 settings = extractSettingsGitLog(log)
 509
 510                 depotPaths = settings['depot-paths']
 511                 change = settings['change']
 512
 513                 changed = False
 514
 515                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 516                                                            for p in depotPaths]))) == 0:
 517                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 518                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 519                     continue
 520
 521                 while change and int(change) > maxChange:
 522                     changed = True
 523                     if self.verbose:
 524                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 525                     system("git update-ref %s \"%s^\"" % (ref, ref))
 526                     log = extractLogMessageFromGitCommit(ref)
 527                     settings =  extractSettingsGitLog(log)
 528
 529
 530                     depotPaths = settings['depot-paths']
 531                     change = settings['change']
 532
 533                 if changed:
 534                     print "%s rewound to %s" % (ref, change)
 535
 536         return True
 537
 538 class P4Submit(Command):
 539     def __init__(self):
 540         Command.__init__(self)
 541         self.options = [
 542                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 543                 optparse.make_option("--origin", dest="origin"),
 544                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 545         ]
 546         self.description = "Submit changes from git to the perforce depot."
 547         self.usage += " [name of git branch to submit into perforce depot]"
 548         self.interactive = True
 549         self.origin = ""
 550         self.detectRename = False
 551         self.verbose = False
 552         self.isWindows = (platform.system() == "Windows")
 553
 554     def check(self):
 555         if len(p4CmdList("opened ...")) > 0:
 556             die("You have files opened with perforce! Close them before starting the sync.")
 557
 558     # replaces everything between 'Description:' and the next P4 submit template field with the
 559     # commit message
 560     def prepareLogMessage(self, template, message):
 561         result = ""
 562
 563         inDescriptionSection = False
 564
 565         for line in template.split("\n"):
 566             if line.startswith("#"):
 567                 result += line + "\n"
 568                 continue
 569
 570             if inDescriptionSection:
 571                 if line.startswith("Files:"):
 572                     inDescriptionSection = False
 573                 else:
 574                     continue
 575             else:
 576                 if line.startswith("Description:"):
 577                     inDescriptionSection = True
 578                     line += "\n"
 579                     for messageLine in message.split("\n"):
 580                         line += "\t" + messageLine + "\n"
 581
 582             result += line + "\n"
 583
 584         return result
 585
 586     def prepareSubmitTemplate(self):
 587         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 588         template = ""
 589         inFilesSection = False
 590         for line in p4_read_pipe_lines("change -o"):
 591             if line.endswith("\r\n"):
 592                 line = line[:-2] + "\n"
 593             if inFilesSection:
 594                 if line.startswith("\t"):
 595                     # path starts and ends with a tab
 596                     path = line[1:]
 597                     lastTab = path.rfind("\t")
 598                     if lastTab != -1:
 599                         path = path[:lastTab]
 600                         if not path.startswith(self.depotPath):
 601                             continue
 602                 else:
 603                     inFilesSection = False
 604             else:
 605                 if line.startswith("Files:"):
 606                     inFilesSection = True
 607
 608             template += line
 609
 610         return template
 611
 612     def applyCommit(self, id):
 613         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 614         diffOpts = ("", "-M")[self.detectRename]
 615         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 616         filesToAdd = set()
 617         filesToDelete = set()
 618         editedFiles = set()
 619         filesToChangeExecBit = {}
 620         for line in diff:
 621             diff = parseDiffTreeEntry(line)
 622             modifier = diff['status']
 623             path = diff['src']
 624             if modifier == "M":
 625                 p4_system("edit \"%s\"" % path)
 626                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 627                     filesToChangeExecBit[path] = diff['dst_mode']
 628                 editedFiles.add(path)
 629             elif modifier == "A":
 630                 filesToAdd.add(path)
 631                 filesToChangeExecBit[path] = diff['dst_mode']
 632                 if path in filesToDelete:
 633                     filesToDelete.remove(path)
 634             elif modifier == "D":
 635                 filesToDelete.add(path)
 636                 if path in filesToAdd:
 637                     filesToAdd.remove(path)
 638             elif modifier == "R":
 639                 src, dest = diff['src'], diff['dst']
 640                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 641                 p4_system("edit \"%s\"" % (dest))
 642                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 643                     filesToChangeExecBit[dest] = diff['dst_mode']
 644                 os.unlink(dest)
 645                 editedFiles.add(dest)
 646                 filesToDelete.add(src)
 647             else:
 648                 die("unknown modifier %s for %s" % (modifier, path))
 649
 650         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 651         patchcmd = diffcmd + " | git apply "
 652         tryPatchCmd = patchcmd + "--check -"
 653         applyPatchCmd = patchcmd + "--check --apply -"
 654
 655         if os.system(tryPatchCmd) != 0:
 656             print "Unfortunately applying the change failed!"
 657             print "What do you want to do?"
 658             response = "x"
 659             while response != "s" and response != "a" and response != "w":
 660                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 661                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 662             if response == "s":
 663                 print "Skipping! Good luck with the next patches..."
 664                 for f in editedFiles:
 665                     p4_system("revert \"%s\"" % f);
 666                 for f in filesToAdd:
 667                     system("rm %s" %f)
 668                 return
 669             elif response == "a":
 670                 os.system(applyPatchCmd)
 671                 if len(filesToAdd) > 0:
 672                     print "You may also want to call p4 add on the following files:"
 673                     print " ".join(filesToAdd)
 674                 if len(filesToDelete):
 675                     print "The following files should be scheduled for deletion with p4 delete:"
 676                     print " ".join(filesToDelete)
 677                 die("Please resolve and submit the conflict manually and "
 678                     + "continue afterwards with git-p4 submit --continue")
 679             elif response == "w":
 680                 system(diffcmd + " > patch.txt")
 681                 print "Patch saved to patch.txt in %s !" % self.clientPath
 682                 die("Please resolve and submit the conflict manually and "
 683                     "continue afterwards with git-p4 submit --continue")
 684
 685         system(applyPatchCmd)
 686
 687         for f in filesToAdd:
 688             p4_system("add \"%s\"" % f)
 689         for f in filesToDelete:
 690             p4_system("revert \"%s\"" % f)
 691             p4_system("delete \"%s\"" % f)
 692
 693         # Set/clear executable bits
 694         for f in filesToChangeExecBit.keys():
 695             mode = filesToChangeExecBit[f]
 696             setP4ExecBit(f, mode)
 697
 698         logMessage = extractLogMessageFromGitCommit(id)
 699         logMessage = logMessage.strip()
 700
 701         template = self.prepareSubmitTemplate()
 702
 703         if self.interactive:
 704             submitTemplate = self.prepareLogMessage(template, logMessage)
 705             if os.environ.has_key("P4DIFF"):
 706                 del(os.environ["P4DIFF"])
 707             diff = p4_read_pipe("diff -du ...")
 708
 709             newdiff = ""
 710             for newFile in filesToAdd:
 711                 newdiff += "==== new file ====\n"
 712                 newdiff += "--- /dev/null\n"
 713                 newdiff += "+++ %s\n" % newFile
 714                 f = open(newFile, "r")
 715                 for line in f.readlines():
 716                     newdiff += "+" + line
 717                 f.close()
 718
 719             separatorLine = "######## everything below this line is just the diff #######\n"
 720
 721             [handle, fileName] = tempfile.mkstemp()
 722             tmpFile = os.fdopen(handle, "w+")
 723             if self.isWindows:
 724                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 725                 separatorLine = separatorLine.replace("\n", "\r\n")
 726                 newdiff = newdiff.replace("\n", "\r\n")
 727             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 728             tmpFile.close()
 729             mtime = os.stat(fileName).st_mtime
 730             defaultEditor = "vi"
 731             if platform.system() == "Windows":
 732                 defaultEditor = "notepad"
 733             if os.environ.has_key("P4EDITOR"):
 734                 editor = os.environ.get("P4EDITOR")
 735             else:
 736                 editor = os.environ.get("EDITOR", defaultEditor);
 737             system(editor + " " + fileName)
 738
 739             response = "y"
 740             if os.stat(fileName).st_mtime <= mtime:
 741                 response = "x"
 742                 while response != "y" and response != "n":
 743                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 744
 745             if response == "y":
 746                 tmpFile = open(fileName, "rb")
 747                 message = tmpFile.read()
 748                 tmpFile.close()
 749                 submitTemplate = message[:message.index(separatorLine)]
 750                 if self.isWindows:
 751                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 752                 p4_write_pipe("submit -i", submitTemplate)
 753             else:
 754                 for f in editedFiles:
 755                     p4_system("revert \"%s\"" % f);
 756                 for f in filesToAdd:
 757                     p4_system("revert \"%s\"" % f);
 758                     system("rm %s" %f)
 759
 760             os.remove(fileName)
 761         else:
 762             fileName = "submit.txt"
 763             file = open(fileName, "w+")
 764             file.write(self.prepareLogMessage(template, logMessage))
 765             file.close()
 766             print ("Perforce submit template written as %s. "
 767                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 768                    % (fileName, fileName))
 769
 770     def run(self, args):
 771         if len(args) == 0:
 772             self.master = currentGitBranch()
 773             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 774                 die("Detecting current git branch failed!")
 775         elif len(args) == 1:
 776             self.master = args[0]
 777         else:
 778             return False
 779
 780         allowSubmit = gitConfig("git-p4.allowSubmit")
 781         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 782             die("%s is not in git-p4.allowSubmit" % self.master)
 783
 784         [upstream, settings] = findUpstreamBranchPoint()
 785         self.depotPath = settings['depot-paths'][0]
 786         if len(self.origin) == 0:
 787             self.origin = upstream
 788
 789         if self.verbose:
 790             print "Origin branch is " + self.origin
 791
 792         if len(self.depotPath) == 0:
 793             print "Internal error: cannot locate perforce depot path from existing branches"
 794             sys.exit(128)
 795
 796         self.clientPath = p4Where(self.depotPath)
 797
 798         if len(self.clientPath) == 0:
 799             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 800             sys.exit(128)
 801
 802         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 803         self.oldWorkingDirectory = os.getcwd()
 804
 805         chdir(self.clientPath)
 806         print "Syncronizing p4 checkout..."
 807         p4_system("sync ...")
 808
 809         self.check()
 810
 811         commits = []
 812         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 813             commits.append(line.strip())
 814         commits.reverse()
 815
 816         while len(commits) > 0:
 817             commit = commits[0]
 818             commits = commits[1:]
 819             self.applyCommit(commit)
 820             if not self.interactive:
 821                 break
 822
 823         if len(commits) == 0:
 824             print "All changes applied!"
 825             chdir(self.oldWorkingDirectory)
 826
 827             sync = P4Sync()
 828             sync.run([])
 829
 830             rebase = P4Rebase()
 831             rebase.rebase()
 832
 833         return True
 834
 835 class P4Sync(Command):
 836     def __init__(self):
 837         Command.__init__(self)
 838         self.options = [
 839                 optparse.make_option("--branch", dest="branch"),
 840                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 841                 optparse.make_option("--changesfile", dest="changesFile"),
 842                 optparse.make_option("--silent", dest="silent", action="store_true"),
 843                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 844                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 845                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 846                                      help="Import into refs/heads/ , not refs/remotes"),
 847                 optparse.make_option("--max-changes", dest="maxChanges"),
 848                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 849                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 850                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 851                                      help="Only sync files that are included in the Perforce Client Spec")
 852         ]
 853         self.description = """Imports from Perforce into a git repository.\n
 854     example:
 855     //depot/my/project/ -- to import the current head
 856     //depot/my/project/@all -- to import everything
 857     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 858
 859     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 860
 861         self.usage += " //depot/path[@revRange]"
 862         self.silent = False
 863         self.createdBranches = Set()
 864         self.committedChanges = Set()
 865         self.branch = ""
 866         self.detectBranches = False
 867         self.detectLabels = False
 868         self.changesFile = ""
 869         self.syncWithOrigin = True
 870         self.verbose = False
 871         self.importIntoRemotes = True
 872         self.maxChanges = ""
 873         self.isWindows = (platform.system() == "Windows")
 874         self.keepRepoPath = False
 875         self.depotPaths = None
 876         self.p4BranchesInGit = []
 877         self.cloneExclude = []
 878         self.useClientSpec = False
 879         self.clientSpecDirs = []
 880
 881         if gitConfig("git-p4.syncFromOrigin") == "false":
 882             self.syncWithOrigin = False
 883
 884     def extractFilesFromCommit(self, commit):
 885         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 886                              for path in self.cloneExclude]
 887         files = []
 888         fnum = 0
 889         while commit.has_key("depotFile%s" % fnum):
 890             path =  commit["depotFile%s" % fnum]
 891
 892             if [p for p in self.cloneExclude
 893                 if path.startswith (p)]:
 894                 found = False
 895             else:
 896                 found = [p for p in self.depotPaths
 897                          if path.startswith (p)]
 898             if not found:
 899                 fnum = fnum + 1
 900                 continue
 901
 902             file = {}
 903             file["path"] = path
 904             file["rev"] = commit["rev%s" % fnum]
 905             file["action"] = commit["action%s" % fnum]
 906             file["type"] = commit["type%s" % fnum]
 907             files.append(file)
 908             fnum = fnum + 1
 909         return files
 910
 911     def stripRepoPath(self, path, prefixes):
 912         if self.keepRepoPath:
 913             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 914
 915         for p in prefixes:
 916             if path.startswith(p):
 917                 path = path[len(p):]
 918
 919         return path
 920
 921     def splitFilesIntoBranches(self, commit):
 922         branches = {}
 923         fnum = 0
 924         while commit.has_key("depotFile%s" % fnum):
 925             path =  commit["depotFile%s" % fnum]
 926             found = [p for p in self.depotPaths
 927                      if path.startswith (p)]
 928             if not found:
 929                 fnum = fnum + 1
 930                 continue
 931
 932             file = {}
 933             file["path"] = path
 934             file["rev"] = commit["rev%s" % fnum]
 935             file["action"] = commit["action%s" % fnum]
 936             file["type"] = commit["type%s" % fnum]
 937             fnum = fnum + 1
 938
 939             relPath = self.stripRepoPath(path, self.depotPaths)
 940
 941             for branch in self.knownBranches.keys():
 942
 943                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 944                 if relPath.startswith(branch + "/"):
 945                     if branch not in branches:
 946                         branches[branch] = []
 947                     branches[branch].append(file)
 948                     break
 949
 950         return branches
 951
 952     ## Should move this out, doesn't use SELF.
 953     def readP4Files(self, files):
 954         filesForCommit = []
 955         filesToRead = []
 956
 957         for f in files:
 958             includeFile = True
 959             for val in self.clientSpecDirs:
 960                 if f['path'].startswith(val[0]):
 961                     if val[1] <= 0:
 962                         includeFile = False
 963                     break
 964
 965             if includeFile:
 966                 filesForCommit.append(f)
 967                 if f['action'] not in ('delete', 'purge'):
 968                     filesToRead.append(f)
 969
 970         filedata = []
 971         if len(filesToRead) > 0:
 972             filedata = p4CmdList('-x - print',
 973                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 974                                                   for f in filesToRead]),
 975                                  stdin_mode='w+')
 976
 977             if "p4ExitCode" in filedata[0]:
 978                 die("Problems executing p4. Error: [%d]."
 979                     % (filedata[0]['p4ExitCode']));
 980
 981         j = 0;
 982         contents = {}
 983         while j < len(filedata):
 984             stat = filedata[j]
 985             j += 1
 986             text = ''
 987             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 988                 text += filedata[j]['data']
 989                 del filedata[j]['data']
 990                 j += 1
 991
 992             if not stat.has_key('depotFile'):
 993                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 994                 continue
 995
 996             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 997                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 998             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 999                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text)
1000
1001             contents[stat['depotFile']] = text
1002
1003         for f in filesForCommit:
1004             path = f['path']
1005             if contents.has_key(path):
1006                 f['data'] = contents[path]
1007
1008         return filesForCommit
1009
1010     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1011         epoch = details["time"]
1012         author = details["user"]
1013
1014         if self.verbose:
1015             print "commit into %s" % branch
1016
1017         # start with reading files; if that fails, we should not
1018         # create a commit.
1019         new_files = []
1020         for f in files:
1021             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1022                 new_files.append (f)
1023             else:
1024                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1025         files = self.readP4Files(new_files)
1026
1027         self.gitStream.write("commit %s\n" % branch)
1028 #        gitStream.write("mark :%s\n" % details["change"])
1029         self.committedChanges.add(int(details["change"]))
1030         committer = ""
1031         if author not in self.users:
1032             self.getUserMapFromPerforceServer()
1033         if author in self.users:
1034             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1035         else:
1036             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1037
1038         self.gitStream.write("committer %s\n" % committer)
1039
1040         self.gitStream.write("data <<EOT\n")
1041         self.gitStream.write(details["desc"])
1042         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1043                              % (','.join (branchPrefixes), details["change"]))
1044         if len(details['options']) > 0:
1045             self.gitStream.write(": options = %s" % details['options'])
1046         self.gitStream.write("]\nEOT\n\n")
1047
1048         if len(parent) > 0:
1049             if self.verbose:
1050                 print "parent %s" % parent
1051             self.gitStream.write("from %s\n" % parent)
1052
1053         for file in files:
1054             if file["type"] == "apple":
1055                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1056                 continue
1057
1058             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1059             if file["action"] in ("delete", "purge"):
1060                 self.gitStream.write("D %s\n" % relPath)
1061             else:
1062                 data = file['data']
1063
1064                 mode = "644"
1065                 if isP4Exec(file["type"]):
1066                     mode = "755"
1067                 elif file["type"] == "symlink":
1068                     mode = "120000"
1069                     # p4 print on a symlink contains "target\n", so strip it off
1070                     data = data[:-1]
1071
1072                 if self.isWindows and file["type"].endswith("text"):
1073                     data = data.replace("\r\n", "\n")
1074
1075                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1076                 self.gitStream.write("data %s\n" % len(data))
1077                 self.gitStream.write(data)
1078                 self.gitStream.write("\n")
1079
1080         self.gitStream.write("\n")
1081
1082         change = int(details["change"])
1083
1084         if self.labels.has_key(change):
1085             label = self.labels[change]
1086             labelDetails = label[0]
1087             labelRevisions = label[1]
1088             if self.verbose:
1089                 print "Change %s is labelled %s" % (change, labelDetails)
1090
1091             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1092                                                     for p in branchPrefixes]))
1093
1094             if len(files) == len(labelRevisions):
1095
1096                 cleanedFiles = {}
1097                 for info in files:
1098                     if info["action"] in ("delete", "purge"):
1099                         continue
1100                     cleanedFiles[info["depotFile"]] = info["rev"]
1101
1102                 if cleanedFiles == labelRevisions:
1103                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1104                     self.gitStream.write("from %s\n" % branch)
1105
1106                     owner = labelDetails["Owner"]
1107                     tagger = ""
1108                     if author in self.users:
1109                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1110                     else:
1111                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1112                     self.gitStream.write("tagger %s\n" % tagger)
1113                     self.gitStream.write("data <<EOT\n")
1114                     self.gitStream.write(labelDetails["Description"])
1115                     self.gitStream.write("EOT\n\n")
1116
1117                 else:
1118                     if not self.silent:
1119                         print ("Tag %s does not match with change %s: files do not match."
1120                                % (labelDetails["label"], change))
1121
1122             else:
1123                 if not self.silent:
1124                     print ("Tag %s does not match with change %s: file count is different."
1125                            % (labelDetails["label"], change))
1126
1127     def getUserCacheFilename(self):
1128         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1129         return home + "/.gitp4-usercache.txt"
1130
1131     def getUserMapFromPerforceServer(self):
1132         if self.userMapFromPerforceServer:
1133             return
1134         self.users = {}
1135
1136         for output in p4CmdList("users"):
1137             if not output.has_key("User"):
1138                 continue
1139             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1140
1141
1142         s = ''
1143         for (key, val) in self.users.items():
1144             s += "%s\t%s\n" % (key, val)
1145
1146         open(self.getUserCacheFilename(), "wb").write(s)
1147         self.userMapFromPerforceServer = True
1148
1149     def loadUserMapFromCache(self):
1150         self.users = {}
1151         self.userMapFromPerforceServer = False
1152         try:
1153             cache = open(self.getUserCacheFilename(), "rb")
1154             lines = cache.readlines()
1155             cache.close()
1156             for line in lines:
1157                 entry = line.strip().split("\t")
1158                 self.users[entry[0]] = entry[1]
1159         except IOError:
1160             self.getUserMapFromPerforceServer()
1161
1162     def getLabels(self):
1163         self.labels = {}
1164
1165         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1166         if len(l) > 0 and not self.silent:
1167             print "Finding files belonging to labels in %s" % `self.depotPaths`
1168
1169         for output in l:
1170             label = output["label"]
1171             revisions = {}
1172             newestChange = 0
1173             if self.verbose:
1174                 print "Querying files for label %s" % label
1175             for file in p4CmdList("files "
1176                                   +  ' '.join (["%s...@%s" % (p, label)
1177                                                 for p in self.depotPaths])):
1178                 revisions[file["depotFile"]] = file["rev"]
1179                 change = int(file["change"])
1180                 if change > newestChange:
1181                     newestChange = change
1182
1183             self.labels[newestChange] = [output, revisions]
1184
1185         if self.verbose:
1186             print "Label changes: %s" % self.labels.keys()
1187
1188     def guessProjectName(self):
1189         for p in self.depotPaths:
1190             if p.endswith("/"):
1191                 p = p[:-1]
1192             p = p[p.strip().rfind("/") + 1:]
1193             if not p.endswith("/"):
1194                p += "/"
1195             return p
1196
1197     def getBranchMapping(self):
1198         lostAndFoundBranches = set()
1199
1200         for info in p4CmdList("branches"):
1201             details = p4Cmd("branch -o %s" % info["branch"])
1202             viewIdx = 0
1203             while details.has_key("View%s" % viewIdx):
1204                 paths = details["View%s" % viewIdx].split(" ")
1205                 viewIdx = viewIdx + 1
1206                 # require standard //depot/foo/... //depot/bar/... mapping
1207                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1208                     continue
1209                 source = paths[0]
1210                 destination = paths[1]
1211                 ## HACK
1212                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1213                     source = source[len(self.depotPaths[0]):-4]
1214                     destination = destination[len(self.depotPaths[0]):-4]
1215
1216                     if destination in self.knownBranches:
1217                         if not self.silent:
1218                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1219                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1220                         continue
1221
1222                     self.knownBranches[destination] = source
1223
1224                     lostAndFoundBranches.discard(destination)
1225
1226                     if source not in self.knownBranches:
1227                         lostAndFoundBranches.add(source)
1228
1229
1230         for branch in lostAndFoundBranches:
1231             self.knownBranches[branch] = branch
1232
1233     def getBranchMappingFromGitBranches(self):
1234         branches = p4BranchesInGit(self.importIntoRemotes)
1235         for branch in branches.keys():
1236             if branch == "master":
1237                 branch = "main"
1238             else:
1239                 branch = branch[len(self.projectName):]
1240             self.knownBranches[branch] = branch
1241
1242     def listExistingP4GitBranches(self):
1243         # branches holds mapping from name to commit
1244         branches = p4BranchesInGit(self.importIntoRemotes)
1245         self.p4BranchesInGit = branches.keys()
1246         for branch in branches.keys():
1247             self.initialParents[self.refPrefix + branch] = branches[branch]
1248
1249     def updateOptionDict(self, d):
1250         option_keys = {}
1251         if self.keepRepoPath:
1252             option_keys['keepRepoPath'] = 1
1253
1254         d["options"] = ' '.join(sorted(option_keys.keys()))
1255
1256     def readOptions(self, d):
1257         self.keepRepoPath = (d.has_key('options')
1258                              and ('keepRepoPath' in d['options']))
1259
1260     def gitRefForBranch(self, branch):
1261         if branch == "main":
1262             return self.refPrefix + "master"
1263
1264         if len(branch) <= 0:
1265             return branch
1266
1267         return self.refPrefix + self.projectName + branch
1268
1269     def gitCommitByP4Change(self, ref, change):
1270         if self.verbose:
1271             print "looking in ref " + ref + " for change %s using bisect..." % change
1272
1273         earliestCommit = ""
1274         latestCommit = parseRevision(ref)
1275
1276         while True:
1277             if self.verbose:
1278                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1279             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1280             if len(next) == 0:
1281                 if self.verbose:
1282                     print "argh"
1283                 return ""
1284             log = extractLogMessageFromGitCommit(next)
1285             settings = extractSettingsGitLog(log)
1286             currentChange = int(settings['change'])
1287             if self.verbose:
1288                 print "current change %s" % currentChange
1289
1290             if currentChange == change:
1291                 if self.verbose:
1292                     print "found %s" % next
1293                 return next
1294
1295             if currentChange < change:
1296                 earliestCommit = "^%s" % next
1297             else:
1298                 latestCommit = "%s" % next
1299
1300         return ""
1301
1302     def importNewBranch(self, branch, maxChange):
1303         # make fast-import flush all changes to disk and update the refs using the checkpoint
1304         # command so that we can try to find the branch parent in the git history
1305         self.gitStream.write("checkpoint\n\n");
1306         self.gitStream.flush();
1307         branchPrefix = self.depotPaths[0] + branch + "/"
1308         range = "@1,%s" % maxChange
1309         #print "prefix" + branchPrefix
1310         changes = p4ChangesForPaths([branchPrefix], range)
1311         if len(changes) <= 0:
1312             return False
1313         firstChange = changes[0]
1314         #print "first change in branch: %s" % firstChange
1315         sourceBranch = self.knownBranches[branch]
1316         sourceDepotPath = self.depotPaths[0] + sourceBranch
1317         sourceRef = self.gitRefForBranch(sourceBranch)
1318         #print "source " + sourceBranch
1319
1320         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1321         #print "branch parent: %s" % branchParentChange
1322         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1323         if len(gitParent) > 0:
1324             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1325             #print "parent git commit: %s" % gitParent
1326
1327         self.importChanges(changes)
1328         return True
1329
1330     def importChanges(self, changes):
1331         cnt = 1
1332         for change in changes:
1333             description = p4Cmd("describe %s" % change)
1334             self.updateOptionDict(description)
1335
1336             if not self.silent:
1337                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1338                 sys.stdout.flush()
1339             cnt = cnt + 1
1340
1341             try:
1342                 if self.detectBranches:
1343                     branches = self.splitFilesIntoBranches(description)
1344                     for branch in branches.keys():
1345                         ## HACK  --hwn
1346                         branchPrefix = self.depotPaths[0] + branch + "/"
1347
1348                         parent = ""
1349
1350                         filesForCommit = branches[branch]
1351
1352                         if self.verbose:
1353                             print "branch is %s" % branch
1354
1355                         self.updatedBranches.add(branch)
1356
1357                         if branch not in self.createdBranches:
1358                             self.createdBranches.add(branch)
1359                             parent = self.knownBranches[branch]
1360                             if parent == branch:
1361                                 parent = ""
1362                             else:
1363                                 fullBranch = self.projectName + branch
1364                                 if fullBranch not in self.p4BranchesInGit:
1365                                     if not self.silent:
1366                                         print("\n    Importing new branch %s" % fullBranch);
1367                                     if self.importNewBranch(branch, change - 1):
1368                                         parent = ""
1369                                         self.p4BranchesInGit.append(fullBranch)
1370                                     if not self.silent:
1371                                         print("\n    Resuming with change %s" % change);
1372
1373                                 if self.verbose:
1374                                     print "parent determined through known branches: %s" % parent
1375
1376                         branch = self.gitRefForBranch(branch)
1377                         parent = self.gitRefForBranch(parent)
1378
1379                         if self.verbose:
1380                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1381
1382                         if len(parent) == 0 and branch in self.initialParents:
1383                             parent = self.initialParents[branch]
1384                             del self.initialParents[branch]
1385
1386                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1387                 else:
1388                     files = self.extractFilesFromCommit(description)
1389                     self.commit(description, files, self.branch, self.depotPaths,
1390                                 self.initialParent)
1391                     self.initialParent = ""
1392             except IOError:
1393                 print self.gitError.read()
1394                 sys.exit(1)
1395
1396     def importHeadRevision(self, revision):
1397         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1398
1399         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1400         details["desc"] = ("Initial import of %s from the state at revision %s"
1401                            % (' '.join(self.depotPaths), revision))
1402         details["change"] = revision
1403         newestRevision = 0
1404
1405         fileCnt = 0
1406         for info in p4CmdList("files "
1407                               +  ' '.join(["%s...%s"
1408                                            % (p, revision)
1409                                            for p in self.depotPaths])):
1410
1411             if info['code'] == 'error':
1412                 sys.stderr.write("p4 returned an error: %s\n"
1413                                  % info['data'])
1414                 sys.exit(1)
1415
1416
1417             change = int(info["change"])
1418             if change > newestRevision:
1419                 newestRevision = change
1420
1421             if info["action"] in ("delete", "purge"):
1422                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1423                 #fileCnt = fileCnt + 1
1424                 continue
1425
1426             for prop in ["depotFile", "rev", "action", "type" ]:
1427                 details["%s%s" % (prop, fileCnt)] = info[prop]
1428
1429             fileCnt = fileCnt + 1
1430
1431         details["change"] = newestRevision
1432         self.updateOptionDict(details)
1433         try:
1434             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1435         except IOError:
1436             print "IO error with git fast-import. Is your git version recent enough?"
1437             print self.gitError.read()
1438
1439
1440     def getClientSpec(self):
1441         specList = p4CmdList( "client -o" )
1442         temp = {}
1443         for entry in specList:
1444             for k,v in entry.iteritems():
1445                 if k.startswith("View"):
1446                     if v.startswith('"'):
1447                         start = 1
1448                     else:
1449                         start = 0
1450                     index = v.find("...")
1451                     v = v[start:index]
1452                     if v.startswith("-"):
1453                         v = v[1:]
1454                         temp[v] = -len(v)
1455                     else:
1456                         temp[v] = len(v)
1457         self.clientSpecDirs = temp.items()
1458         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1459
1460     def run(self, args):
1461         self.depotPaths = []
1462         self.changeRange = ""
1463         self.initialParent = ""
1464         self.previousDepotPaths = []
1465
1466         # map from branch depot path to parent branch
1467         self.knownBranches = {}
1468         self.initialParents = {}
1469         self.hasOrigin = originP4BranchesExist()
1470         if not self.syncWithOrigin:
1471             self.hasOrigin = False
1472
1473         if self.importIntoRemotes:
1474             self.refPrefix = "refs/remotes/p4/"
1475         else:
1476             self.refPrefix = "refs/heads/p4/"
1477
1478         if self.syncWithOrigin and self.hasOrigin:
1479             if not self.silent:
1480                 print "Syncing with origin first by calling git fetch origin"
1481             system("git fetch origin")
1482
1483         if len(self.branch) == 0:
1484             self.branch = self.refPrefix + "master"
1485             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1486                 system("git update-ref %s refs/heads/p4" % self.branch)
1487                 system("git branch -D p4");
1488             # create it /after/ importing, when master exists
1489             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1490                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1491
1492         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1493             self.getClientSpec()
1494
1495         # TODO: should always look at previous commits,
1496         # merge with previous imports, if possible.
1497         if args == []:
1498             if self.hasOrigin:
1499                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1500             self.listExistingP4GitBranches()
1501
1502             if len(self.p4BranchesInGit) > 1:
1503                 if not self.silent:
1504                     print "Importing from/into multiple branches"
1505                 self.detectBranches = True
1506
1507             if self.verbose:
1508                 print "branches: %s" % self.p4BranchesInGit
1509
1510             p4Change = 0
1511             for branch in self.p4BranchesInGit:
1512                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1513
1514                 settings = extractSettingsGitLog(logMsg)
1515
1516                 self.readOptions(settings)
1517                 if (settings.has_key('depot-paths')
1518                     and settings.has_key ('change')):
1519                     change = int(settings['change']) + 1
1520                     p4Change = max(p4Change, change)
1521
1522                     depotPaths = sorted(settings['depot-paths'])
1523                     if self.previousDepotPaths == []:
1524                         self.previousDepotPaths = depotPaths
1525                     else:
1526                         paths = []
1527                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1528                             for i in range(0, min(len(cur), len(prev))):
1529                                 if cur[i] <> prev[i]:
1530                                     i = i - 1
1531                                     break
1532
1533                             paths.append (cur[:i + 1])
1534
1535                         self.previousDepotPaths = paths
1536
1537             if p4Change > 0:
1538                 self.depotPaths = sorted(self.previousDepotPaths)
1539                 self.changeRange = "@%s,#head" % p4Change
1540                 if not self.detectBranches:
1541                     self.initialParent = parseRevision(self.branch)
1542                 if not self.silent and not self.detectBranches:
1543                     print "Performing incremental import into %s git branch" % self.branch
1544
1545         if not self.branch.startswith("refs/"):
1546             self.branch = "refs/heads/" + self.branch
1547
1548         if len(args) == 0 and self.depotPaths:
1549             if not self.silent:
1550                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1551         else:
1552             if self.depotPaths and self.depotPaths != args:
1553                 print ("previous import used depot path %s and now %s was specified. "
1554                        "This doesn't work!" % (' '.join (self.depotPaths),
1555                                                ' '.join (args)))
1556                 sys.exit(1)
1557
1558             self.depotPaths = sorted(args)
1559
1560         revision = ""
1561         self.users = {}
1562
1563         newPaths = []
1564         for p in self.depotPaths:
1565             if p.find("@") != -1:
1566                 atIdx = p.index("@")
1567                 self.changeRange = p[atIdx:]
1568                 if self.changeRange == "@all":
1569                     self.changeRange = ""
1570                 elif ',' not in self.changeRange:
1571                     revision = self.changeRange
1572                     self.changeRange = ""
1573                 p = p[:atIdx]
1574             elif p.find("#") != -1:
1575                 hashIdx = p.index("#")
1576                 revision = p[hashIdx:]
1577                 p = p[:hashIdx]
1578             elif self.previousDepotPaths == []:
1579                 revision = "#head"
1580
1581             p = re.sub ("\.\.\.$", "", p)
1582             if not p.endswith("/"):
1583                 p += "/"
1584
1585             newPaths.append(p)
1586
1587         self.depotPaths = newPaths
1588
1589
1590         self.loadUserMapFromCache()
1591         self.labels = {}
1592         if self.detectLabels:
1593             self.getLabels();
1594
1595         if self.detectBranches:
1596             ## FIXME - what's a P4 projectName ?
1597             self.projectName = self.guessProjectName()
1598
1599             if self.hasOrigin:
1600                 self.getBranchMappingFromGitBranches()
1601             else:
1602                 self.getBranchMapping()
1603             if self.verbose:
1604                 print "p4-git branches: %s" % self.p4BranchesInGit
1605                 print "initial parents: %s" % self.initialParents
1606             for b in self.p4BranchesInGit:
1607                 if b != "master":
1608
1609                     ## FIXME
1610                     b = b[len(self.projectName):]
1611                 self.createdBranches.add(b)
1612
1613         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1614
1615         importProcess = subprocess.Popen(["git", "fast-import"],
1616                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1617                                          stderr=subprocess.PIPE);
1618         self.gitOutput = importProcess.stdout
1619         self.gitStream = importProcess.stdin
1620         self.gitError = importProcess.stderr
1621
1622         if revision:
1623             self.importHeadRevision(revision)
1624         else:
1625             changes = []
1626
1627             if len(self.changesFile) > 0:
1628                 output = open(self.changesFile).readlines()
1629                 changeSet = Set()
1630                 for line in output:
1631                     changeSet.add(int(line))
1632
1633                 for change in changeSet:
1634                     changes.append(change)
1635
1636                 changes.sort()
1637             else:
1638                 if self.verbose:
1639                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1640                                                               self.changeRange)
1641                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1642
1643                 if len(self.maxChanges) > 0:
1644                     changes = changes[:min(int(self.maxChanges), len(changes))]
1645
1646             if len(changes) == 0:
1647                 if not self.silent:
1648                     print "No changes to import!"
1649                 return True
1650
1651             if not self.silent and not self.detectBranches:
1652                 print "Import destination: %s" % self.branch
1653
1654             self.updatedBranches = set()
1655
1656             self.importChanges(changes)
1657
1658             if not self.silent:
1659                 print ""
1660                 if len(self.updatedBranches) > 0:
1661                     sys.stdout.write("Updated branches: ")
1662                     for b in self.updatedBranches:
1663                         sys.stdout.write("%s " % b)
1664                     sys.stdout.write("\n")
1665
1666         self.gitStream.close()
1667         if importProcess.wait() != 0:
1668             die("fast-import failed: %s" % self.gitError.read())
1669         self.gitOutput.close()
1670         self.gitError.close()
1671
1672         return True
1673
1674 class P4Rebase(Command):
1675     def __init__(self):
1676         Command.__init__(self)
1677         self.options = [ ]
1678         self.description = ("Fetches the latest revision from perforce and "
1679                             + "rebases the current work (branch) against it")
1680         self.verbose = False
1681
1682     def run(self, args):
1683         sync = P4Sync()
1684         sync.run([])
1685
1686         return self.rebase()
1687
1688     def rebase(self):
1689         if os.system("git update-index --refresh") != 0:
1690             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1691         if len(read_pipe("git diff-index HEAD --")) > 0:
1692             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1693
1694         [upstream, settings] = findUpstreamBranchPoint()
1695         if len(upstream) == 0:
1696             die("Cannot find upstream branchpoint for rebase")
1697
1698         # the branchpoint may be p4/foo~3, so strip off the parent
1699         upstream = re.sub("~[0-9]+$", "", upstream)
1700
1701         print "Rebasing the current branch onto %s" % upstream
1702         oldHead = read_pipe("git rev-parse HEAD").strip()
1703         system("git rebase %s" % upstream)
1704         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1705         return True
1706
1707 class P4Clone(P4Sync):
1708     def __init__(self):
1709         P4Sync.__init__(self)
1710         self.description = "Creates a new git repository and imports from Perforce into it"
1711         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1712         self.options += [
1713             optparse.make_option("--destination", dest="cloneDestination",
1714                                  action='store', default=None,
1715                                  help="where to leave result of the clone"),
1716             optparse.make_option("-/", dest="cloneExclude",
1717                                  action="append", type="string",
1718                                  help="exclude depot path")
1719         ]
1720         self.cloneDestination = None
1721         self.needsGit = False
1722
1723     # This is required for the "append" cloneExclude action
1724     def ensure_value(self, attr, value):
1725         if not hasattr(self, attr) or getattr(self, attr) is None:
1726             setattr(self, attr, value)
1727         return getattr(self, attr)
1728
1729     def defaultDestination(self, args):
1730         ## TODO: use common prefix of args?
1731         depotPath = args[0]
1732         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1733         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1734         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1735         depotDir = re.sub(r"/$", "", depotDir)
1736         return os.path.split(depotDir)[1]
1737
1738     def run(self, args):
1739         if len(args) < 1:
1740             return False
1741
1742         if self.keepRepoPath and not self.cloneDestination:
1743             sys.stderr.write("Must specify destination for --keep-path\n")
1744             sys.exit(1)
1745
1746         depotPaths = args
1747
1748         if not self.cloneDestination and len(depotPaths) > 1:
1749             self.cloneDestination = depotPaths[-1]
1750             depotPaths = depotPaths[:-1]
1751
1752         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1753         for p in depotPaths:
1754             if not p.startswith("//"):
1755                 return False
1756
1757         if not self.cloneDestination:
1758             self.cloneDestination = self.defaultDestination(args)
1759
1760         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1761         if not os.path.exists(self.cloneDestination):
1762             os.makedirs(self.cloneDestination)
1763         chdir(self.cloneDestination)
1764         system("git init")
1765         self.gitdir = os.getcwd() + "/.git"
1766         if not P4Sync.run(self, depotPaths):
1767             return False
1768         if self.branch != "master":
1769             if self.importIntoRemotes:
1770                 masterbranch = "refs/remotes/p4/master"
1771             else:
1772                 masterbranch = "refs/heads/p4/master"
1773             if gitBranchExists(masterbranch):
1774                 system("git branch master %s" % masterbranch)
1775                 system("git checkout -f")
1776             else:
1777                 print "Could not detect main branch. No checkout/master branch created."
1778
1779         return True
1780
1781 class P4Branches(Command):
1782     def __init__(self):
1783         Command.__init__(self)
1784         self.options = [ ]
1785         self.description = ("Shows the git branches that hold imports and their "
1786                             + "corresponding perforce depot paths")
1787         self.verbose = False
1788
1789     def run(self, args):
1790         if originP4BranchesExist():
1791             createOrUpdateBranchesFromOrigin()
1792
1793         cmdline = "git rev-parse --symbolic "
1794         cmdline += " --remotes"
1795
1796         for line in read_pipe_lines(cmdline):
1797             line = line.strip()
1798
1799             if not line.startswith('p4/') or line == "p4/HEAD":
1800                 continue
1801             branch = line
1802
1803             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1804             settings = extractSettingsGitLog(log)
1805
1806             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1807         return True
1808
1809 class HelpFormatter(optparse.IndentedHelpFormatter):
1810     def __init__(self):
1811         optparse.IndentedHelpFormatter.__init__(self)
1812
1813     def format_description(self, description):
1814         if description:
1815             return description + "\n"
1816         else:
1817             return ""
1818
1819 def printUsage(commands):
1820     print "usage: %s <command> [options]" % sys.argv[0]
1821     print ""
1822     print "valid commands: %s" % ", ".join(commands)
1823     print ""
1824     print "Try %s <command> --help for command specific help." % sys.argv[0]
1825     print ""
1826
1827 commands = {
1828     "debug" : P4Debug,
1829     "submit" : P4Submit,
1830     "commit" : P4Submit,
1831     "sync" : P4Sync,
1832     "rebase" : P4Rebase,
1833     "clone" : P4Clone,
1834     "rollback" : P4RollBack,
1835     "branches" : P4Branches
1836 }
1837
1838
1839 def main():
1840     if len(sys.argv[1:]) == 0:
1841         printUsage(commands.keys())
1842         sys.exit(2)
1843
1844     cmd = ""
1845     cmdName = sys.argv[1]
1846     try:
1847         klass = commands[cmdName]
1848         cmd = klass()
1849     except KeyError:
1850         print "unknown command %s" % cmdName
1851         print ""
1852         printUsage(commands.keys())
1853         sys.exit(2)
1854
1855     options = cmd.options
1856     cmd.gitdir = os.environ.get("GIT_DIR", None)
1857
1858     args = sys.argv[2:]
1859
1860     if len(options) > 0:
1861         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1862
1863         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1864                                        options,
1865                                        description = cmd.description,
1866                                        formatter = HelpFormatter())
1867
1868         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1869     global verbose
1870     verbose = cmd.verbose
1871     if cmd.needsGit:
1872         if cmd.gitdir == None:
1873             cmd.gitdir = os.path.abspath(".git")
1874             if not isValidGitDir(cmd.gitdir):
1875                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1876                 if os.path.exists(cmd.gitdir):
1877                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1878                     if len(cdup) > 0:
1879                         chdir(cdup);
1880
1881         if not isValidGitDir(cmd.gitdir):
1882             if isValidGitDir(cmd.gitdir + "/.git"):
1883                 cmd.gitdir += "/.git"
1884             else:
1885                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1886
1887         os.environ["GIT_DIR"] = cmd.gitdir
1888
1889     if not cmd.run(args):
1890         parser.print_help()
1891
1892
1893 if __name__ == '__main__':
1894     main()