contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     if os.name == 'nt':
  56         os.environ['PWD']=dir
  57     os.chdir(dir)
  58
  59 def die(msg):
  60     if verbose:
  61         raise Exception(msg)
  62     else:
  63         sys.stderr.write(msg + "\n")
  64         sys.exit(1)
  65
  66 def write_pipe(c, str):
  67     if verbose:
  68         sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70     pipe = os.popen(c, 'w')
  71     val = pipe.write(str)
  72     if pipe.close():
  73         die('Command failed: %s' % c)
  74
  75     return val
  76
  77 def p4_write_pipe(c, str):
  78     real_cmd = p4_build_cmd(c)
  79     return write_pipe(real_cmd, str)
  80
  81 def read_pipe(c, ignore_error=False):
  82     if verbose:
  83         sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85     pipe = os.popen(c, 'rb')
  86     val = pipe.read()
  87     if pipe.close() and not ignore_error:
  88         die('Command failed: %s' % c)
  89
  90     return val
  91
  92 def p4_read_pipe(c, ignore_error=False):
  93     real_cmd = p4_build_cmd(c)
  94     return read_pipe(real_cmd, ignore_error)
  95
  96 def read_pipe_lines(c):
  97     if verbose:
  98         sys.stderr.write('Reading pipe: %s\n' % c)
  99     ## todo: check return status
 100     pipe = os.popen(c, 'rb')
 101     val = pipe.readlines()
 102     if pipe.close():
 103         die('Command failed: %s' % c)
 104
 105     return val
 106
 107 def p4_read_pipe_lines(c):
 108     """Specifically invoke p4 on the command supplied. """
 109     real_cmd = p4_build_cmd(c)
 110     return read_pipe_lines(real_cmd)
 111
 112 def system(cmd):
 113     if verbose:
 114         sys.stderr.write("executing %s\n" % cmd)
 115     if os.system(cmd) != 0:
 116         die("command failed: %s" % cmd)
 117
 118 def p4_system(cmd):
 119     """Specifically invoke p4 as the system command. """
 120     real_cmd = p4_build_cmd(cmd)
 121     return system(real_cmd)
 122
 123 def isP4Exec(kind):
 124     """Determine if a Perforce 'kind' should have execute permission
 125
 126     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128     a plus sign, it is also executable"""
 129     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131 def setP4ExecBit(file, mode):
 132     # Reopens an already open file and changes the execute bit to match
 133     # the execute bit setting in the passed in mode.
 134
 135     p4Type = "+x"
 136
 137     if not isModeExec(mode):
 138         p4Type = getP4OpenedType(file)
 139         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141         if p4Type[-1] == "+":
 142             p4Type = p4Type[0:-1]
 143
 144     p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146 def getP4OpenedType(file):
 147     # Returns the perforce file type for the given file.
 148
 149     result = p4_read_pipe("opened %s" % file)
 150     match = re.match(".*\((.+)\)\r?$", result)
 151     if match:
 152         return match.group(1)
 153     else:
 154         die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156 def diffTreePattern():
 157     # This is a simple generator for the diff tree regex pattern. This could be
 158     # a class variable if this and parseDiffTreeEntry were a part of a class.
 159     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160     while True:
 161         yield pattern
 162
 163 def parseDiffTreeEntry(entry):
 164     """Parses a single diff tree entry into its component elements.
 165
 166     See git-diff-tree(1) manpage for details about the format of the diff
 167     output. This method returns a dictionary with the following elements:
 168
 169     src_mode - The mode of the source file
 170     dst_mode - The mode of the destination file
 171     src_sha1 - The sha1 for the source file
 172     dst_sha1 - The sha1 fr the destination file
 173     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174     status_score - The score for the status (applicable for 'C' and 'R'
 175                    statuses). This is None if there is no score.
 176     src - The path for the source file.
 177     dst - The path for the destination file. This is only present for
 178           copy or renames. If it is not present, this is None.
 179
 180     If the pattern is not matched, None is returned."""
 181
 182     match = diffTreePattern().next().match(entry)
 183     if match:
 184         return {
 185             'src_mode': match.group(1),
 186             'dst_mode': match.group(2),
 187             'src_sha1': match.group(3),
 188             'dst_sha1': match.group(4),
 189             'status': match.group(5),
 190             'status_score': match.group(6),
 191             'src': match.group(7),
 192             'dst': match.group(10)
 193         }
 194     return None
 195
 196 def isModeExec(mode):
 197     # Returns True if the given git mode represents an executable file,
 198     # otherwise False.
 199     return mode[-3:] == "755"
 200
 201 def isModeExecChanged(src_mode, dst_mode):
 202     return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205     cmd = p4_build_cmd("-G %s" % (cmd))
 206     if verbose:
 207         sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209     # Use a temporary file to avoid deadlocks without
 210     # subprocess.communicate(), which would put another copy
 211     # of stdout into memory.
 212     stdin_file = None
 213     if stdin is not None:
 214         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215         stdin_file.write(stdin)
 216         stdin_file.flush()
 217         stdin_file.seek(0)
 218
 219     p4 = subprocess.Popen(cmd, shell=True,
 220                           stdin=stdin_file,
 221                           stdout=subprocess.PIPE)
 222
 223     result = []
 224     try:
 225         while True:
 226             entry = marshal.load(p4.stdout)
 227             result.append(entry)
 228     except EOFError:
 229         pass
 230     exitCode = p4.wait()
 231     if exitCode != 0:
 232         entry = {}
 233         entry["p4ExitCode"] = exitCode
 234         result.append(entry)
 235
 236     return result
 237
 238 def p4Cmd(cmd):
 239     list = p4CmdList(cmd)
 240     result = {}
 241     for entry in list:
 242         result.update(entry)
 243     return result;
 244
 245 def p4Where(depotPath):
 246     if not depotPath.endswith("/"):
 247         depotPath += "/"
 248     depotPath = depotPath + "..."
 249     outputList = p4CmdList("where %s" % depotPath)
 250     output = None
 251     for entry in outputList:
 252         if "depotFile" in entry:
 253             if entry["depotFile"] == depotPath:
 254                 output = entry
 255                 break
 256         elif "data" in entry:
 257             data = entry.get("data")
 258             space = data.find(" ")
 259             if data[:space] == depotPath:
 260                 output = entry
 261                 break
 262     if output == None:
 263         return ""
 264     if output["code"] == "error":
 265         return ""
 266     clientPath = ""
 267     if "path" in output:
 268         clientPath = output.get("path")
 269     elif "data" in output:
 270         data = output.get("data")
 271         lastSpace = data.rfind(" ")
 272         clientPath = data[lastSpace + 1:]
 273
 274     if clientPath.endswith("..."):
 275         clientPath = clientPath[:-3]
 276     return clientPath
 277
 278 def currentGitBranch():
 279     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 280
 281 def isValidGitDir(path):
 282     if (os.path.exists(path + "/HEAD")
 283         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 284         return True;
 285     return False
 286
 287 def parseRevision(ref):
 288     return read_pipe("git rev-parse %s" % ref).strip()
 289
 290 def extractLogMessageFromGitCommit(commit):
 291     logMessage = ""
 292
 293     ## fixme: title is first line of commit, not 1st paragraph.
 294     foundTitle = False
 295     for log in read_pipe_lines("git cat-file commit %s" % commit):
 296        if not foundTitle:
 297            if len(log) == 1:
 298                foundTitle = True
 299            continue
 300
 301        logMessage += log
 302     return logMessage
 303
 304 def extractSettingsGitLog(log):
 305     values = {}
 306     for line in log.split("\n"):
 307         line = line.strip()
 308         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 309         if not m:
 310             continue
 311
 312         assignments = m.group(1).split (':')
 313         for a in assignments:
 314             vals = a.split ('=')
 315             key = vals[0].strip()
 316             val = ('='.join (vals[1:])).strip()
 317             if val.endswith ('\"') and val.startswith('"'):
 318                 val = val[1:-1]
 319
 320             values[key] = val
 321
 322     paths = values.get("depot-paths")
 323     if not paths:
 324         paths = values.get("depot-path")
 325     if paths:
 326         values['depot-paths'] = paths.split(',')
 327     return values
 328
 329 def gitBranchExists(branch):
 330     proc = subprocess.Popen(["git", "rev-parse", branch],
 331                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 332     return proc.wait() == 0;
 333
 334 _gitConfig = {}
 335 def gitConfig(key):
 336     if not _gitConfig.has_key(key):
 337         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 338     return _gitConfig[key]
 339
 340 def p4BranchesInGit(branchesAreInRemotes = True):
 341     branches = {}
 342
 343     cmdline = "git rev-parse --symbolic "
 344     if branchesAreInRemotes:
 345         cmdline += " --remotes"
 346     else:
 347         cmdline += " --branches"
 348
 349     for line in read_pipe_lines(cmdline):
 350         line = line.strip()
 351
 352         ## only import to p4/
 353         if not line.startswith('p4/') or line == "p4/HEAD":
 354             continue
 355         branch = line
 356
 357         # strip off p4
 358         branch = re.sub ("^p4/", "", line)
 359
 360         branches[branch] = parseRevision(line)
 361     return branches
 362
 363 def findUpstreamBranchPoint(head = "HEAD"):
 364     branches = p4BranchesInGit()
 365     # map from depot-path to branch name
 366     branchByDepotPath = {}
 367     for branch in branches.keys():
 368         tip = branches[branch]
 369         log = extractLogMessageFromGitCommit(tip)
 370         settings = extractSettingsGitLog(log)
 371         if settings.has_key("depot-paths"):
 372             paths = ",".join(settings["depot-paths"])
 373             branchByDepotPath[paths] = "remotes/p4/" + branch
 374
 375     settings = None
 376     parent = 0
 377     while parent < 65535:
 378         commit = head + "~%s" % parent
 379         log = extractLogMessageFromGitCommit(commit)
 380         settings = extractSettingsGitLog(log)
 381         if settings.has_key("depot-paths"):
 382             paths = ",".join(settings["depot-paths"])
 383             if branchByDepotPath.has_key(paths):
 384                 return [branchByDepotPath[paths], settings]
 385
 386         parent = parent + 1
 387
 388     return ["", settings]
 389
 390 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 391     if not silent:
 392         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 393                % localRefPrefix)
 394
 395     originPrefix = "origin/p4/"
 396
 397     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 398         line = line.strip()
 399         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 400             continue
 401
 402         headName = line[len(originPrefix):]
 403         remoteHead = localRefPrefix + headName
 404         originHead = line
 405
 406         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 407         if (not original.has_key('depot-paths')
 408             or not original.has_key('change')):
 409             continue
 410
 411         update = False
 412         if not gitBranchExists(remoteHead):
 413             if verbose:
 414                 print "creating %s" % remoteHead
 415             update = True
 416         else:
 417             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 418             if settings.has_key('change') > 0:
 419                 if settings['depot-paths'] == original['depot-paths']:
 420                     originP4Change = int(original['change'])
 421                     p4Change = int(settings['change'])
 422                     if originP4Change > p4Change:
 423                         print ("%s (%s) is newer than %s (%s). "
 424                                "Updating p4 branch from origin."
 425                                % (originHead, originP4Change,
 426                                   remoteHead, p4Change))
 427                         update = True
 428                 else:
 429                     print ("Ignoring: %s was imported from %s while "
 430                            "%s was imported from %s"
 431                            % (originHead, ','.join(original['depot-paths']),
 432                               remoteHead, ','.join(settings['depot-paths'])))
 433
 434         if update:
 435             system("git update-ref %s %s" % (remoteHead, originHead))
 436
 437 def originP4BranchesExist():
 438         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 439
 440 def p4ChangesForPaths(depotPaths, changeRange):
 441     assert depotPaths
 442     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 443                                                         for p in depotPaths]))
 444
 445     changes = {}
 446     for line in output:
 447         changeNum = int(line.split(" ")[1])
 448         changes[changeNum] = True
 449
 450     changelist = changes.keys()
 451     changelist.sort()
 452     return changelist
 453
 454 class Command:
 455     def __init__(self):
 456         self.usage = "usage: %prog [options]"
 457         self.needsGit = True
 458
 459 class P4Debug(Command):
 460     def __init__(self):
 461         Command.__init__(self)
 462         self.options = [
 463             optparse.make_option("--verbose", dest="verbose", action="store_true",
 464                                  default=False),
 465             ]
 466         self.description = "A tool to debug the output of p4 -G."
 467         self.needsGit = False
 468         self.verbose = False
 469
 470     def run(self, args):
 471         j = 0
 472         for output in p4CmdList(" ".join(args)):
 473             print 'Element: %d' % j
 474             j += 1
 475             print output
 476         return True
 477
 478 class P4RollBack(Command):
 479     def __init__(self):
 480         Command.__init__(self)
 481         self.options = [
 482             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 483             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 484         ]
 485         self.description = "A tool to debug the multi-branch import. Don't use :)"
 486         self.verbose = False
 487         self.rollbackLocalBranches = False
 488
 489     def run(self, args):
 490         if len(args) != 1:
 491             return False
 492         maxChange = int(args[0])
 493
 494         if "p4ExitCode" in p4Cmd("changes -m 1"):
 495             die("Problems executing p4");
 496
 497         if self.rollbackLocalBranches:
 498             refPrefix = "refs/heads/"
 499             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 500         else:
 501             refPrefix = "refs/remotes/"
 502             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 503
 504         for line in lines:
 505             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 506                 line = line.strip()
 507                 ref = refPrefix + line
 508                 log = extractLogMessageFromGitCommit(ref)
 509                 settings = extractSettingsGitLog(log)
 510
 511                 depotPaths = settings['depot-paths']
 512                 change = settings['change']
 513
 514                 changed = False
 515
 516                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 517                                                            for p in depotPaths]))) == 0:
 518                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 519                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 520                     continue
 521
 522                 while change and int(change) > maxChange:
 523                     changed = True
 524                     if self.verbose:
 525                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 526                     system("git update-ref %s \"%s^\"" % (ref, ref))
 527                     log = extractLogMessageFromGitCommit(ref)
 528                     settings =  extractSettingsGitLog(log)
 529
 530
 531                     depotPaths = settings['depot-paths']
 532                     change = settings['change']
 533
 534                 if changed:
 535                     print "%s rewound to %s" % (ref, change)
 536
 537         return True
 538
 539 class P4Submit(Command):
 540     def __init__(self):
 541         Command.__init__(self)
 542         self.options = [
 543                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 544                 optparse.make_option("--origin", dest="origin"),
 545                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 546         ]
 547         self.description = "Submit changes from git to the perforce depot."
 548         self.usage += " [name of git branch to submit into perforce depot]"
 549         self.interactive = True
 550         self.origin = ""
 551         self.detectRename = False
 552         self.verbose = False
 553         self.isWindows = (platform.system() == "Windows")
 554
 555     def check(self):
 556         if len(p4CmdList("opened ...")) > 0:
 557             die("You have files opened with perforce! Close them before starting the sync.")
 558
 559     # replaces everything between 'Description:' and the next P4 submit template field with the
 560     # commit message
 561     def prepareLogMessage(self, template, message):
 562         result = ""
 563
 564         inDescriptionSection = False
 565
 566         for line in template.split("\n"):
 567             if line.startswith("#"):
 568                 result += line + "\n"
 569                 continue
 570
 571             if inDescriptionSection:
 572                 if line.startswith("Files:"):
 573                     inDescriptionSection = False
 574                 else:
 575                     continue
 576             else:
 577                 if line.startswith("Description:"):
 578                     inDescriptionSection = True
 579                     line += "\n"
 580                     for messageLine in message.split("\n"):
 581                         line += "\t" + messageLine + "\n"
 582
 583             result += line + "\n"
 584
 585         return result
 586
 587     def prepareSubmitTemplate(self):
 588         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 589         template = ""
 590         inFilesSection = False
 591         for line in p4_read_pipe_lines("change -o"):
 592             if line.endswith("\r\n"):
 593                 line = line[:-2] + "\n"
 594             if inFilesSection:
 595                 if line.startswith("\t"):
 596                     # path starts and ends with a tab
 597                     path = line[1:]
 598                     lastTab = path.rfind("\t")
 599                     if lastTab != -1:
 600                         path = path[:lastTab]
 601                         if not path.startswith(self.depotPath):
 602                             continue
 603                 else:
 604                     inFilesSection = False
 605             else:
 606                 if line.startswith("Files:"):
 607                     inFilesSection = True
 608
 609             template += line
 610
 611         return template
 612
 613     def applyCommit(self, id):
 614         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 615         diffOpts = ("", "-M")[self.detectRename]
 616         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 617         filesToAdd = set()
 618         filesToDelete = set()
 619         editedFiles = set()
 620         filesToChangeExecBit = {}
 621         for line in diff:
 622             diff = parseDiffTreeEntry(line)
 623             modifier = diff['status']
 624             path = diff['src']
 625             if modifier == "M":
 626                 p4_system("edit \"%s\"" % path)
 627                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 628                     filesToChangeExecBit[path] = diff['dst_mode']
 629                 editedFiles.add(path)
 630             elif modifier == "A":
 631                 filesToAdd.add(path)
 632                 filesToChangeExecBit[path] = diff['dst_mode']
 633                 if path in filesToDelete:
 634                     filesToDelete.remove(path)
 635             elif modifier == "D":
 636                 filesToDelete.add(path)
 637                 if path in filesToAdd:
 638                     filesToAdd.remove(path)
 639             elif modifier == "R":
 640                 src, dest = diff['src'], diff['dst']
 641                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 642                 p4_system("edit \"%s\"" % (dest))
 643                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 644                     filesToChangeExecBit[dest] = diff['dst_mode']
 645                 os.unlink(dest)
 646                 editedFiles.add(dest)
 647                 filesToDelete.add(src)
 648             else:
 649                 die("unknown modifier %s for %s" % (modifier, path))
 650
 651         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 652         patchcmd = diffcmd + " | git apply "
 653         tryPatchCmd = patchcmd + "--check -"
 654         applyPatchCmd = patchcmd + "--check --apply -"
 655
 656         if os.system(tryPatchCmd) != 0:
 657             print "Unfortunately applying the change failed!"
 658             print "What do you want to do?"
 659             response = "x"
 660             while response != "s" and response != "a" and response != "w":
 661                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 662                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 663             if response == "s":
 664                 print "Skipping! Good luck with the next patches..."
 665                 for f in editedFiles:
 666                     p4_system("revert \"%s\"" % f);
 667                 for f in filesToAdd:
 668                     system("rm %s" %f)
 669                 return
 670             elif response == "a":
 671                 os.system(applyPatchCmd)
 672                 if len(filesToAdd) > 0:
 673                     print "You may also want to call p4 add on the following files:"
 674                     print " ".join(filesToAdd)
 675                 if len(filesToDelete):
 676                     print "The following files should be scheduled for deletion with p4 delete:"
 677                     print " ".join(filesToDelete)
 678                 die("Please resolve and submit the conflict manually and "
 679                     + "continue afterwards with git-p4 submit --continue")
 680             elif response == "w":
 681                 system(diffcmd + " > patch.txt")
 682                 print "Patch saved to patch.txt in %s !" % self.clientPath
 683                 die("Please resolve and submit the conflict manually and "
 684                     "continue afterwards with git-p4 submit --continue")
 685
 686         system(applyPatchCmd)
 687
 688         for f in filesToAdd:
 689             p4_system("add \"%s\"" % f)
 690         for f in filesToDelete:
 691             p4_system("revert \"%s\"" % f)
 692             p4_system("delete \"%s\"" % f)
 693
 694         # Set/clear executable bits
 695         for f in filesToChangeExecBit.keys():
 696             mode = filesToChangeExecBit[f]
 697             setP4ExecBit(f, mode)
 698
 699         logMessage = extractLogMessageFromGitCommit(id)
 700         logMessage = logMessage.strip()
 701
 702         template = self.prepareSubmitTemplate()
 703
 704         if self.interactive:
 705             submitTemplate = self.prepareLogMessage(template, logMessage)
 706             if os.environ.has_key("P4DIFF"):
 707                 del(os.environ["P4DIFF"])
 708             diff = p4_read_pipe("diff -du ...")
 709
 710             newdiff = ""
 711             for newFile in filesToAdd:
 712                 newdiff += "==== new file ====\n"
 713                 newdiff += "--- /dev/null\n"
 714                 newdiff += "+++ %s\n" % newFile
 715                 f = open(newFile, "r")
 716                 for line in f.readlines():
 717                     newdiff += "+" + line
 718                 f.close()
 719
 720             separatorLine = "######## everything below this line is just the diff #######\n"
 721
 722             [handle, fileName] = tempfile.mkstemp()
 723             tmpFile = os.fdopen(handle, "w+")
 724             if self.isWindows:
 725                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 726                 separatorLine = separatorLine.replace("\n", "\r\n")
 727                 newdiff = newdiff.replace("\n", "\r\n")
 728             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 729             tmpFile.close()
 730             mtime = os.stat(fileName).st_mtime
 731             defaultEditor = "vi"
 732             if platform.system() == "Windows":
 733                 defaultEditor = "notepad"
 734             if os.environ.has_key("P4EDITOR"):
 735                 editor = os.environ.get("P4EDITOR")
 736             else:
 737                 editor = os.environ.get("EDITOR", defaultEditor);
 738             system(editor + " " + fileName)
 739
 740             response = "y"
 741             if os.stat(fileName).st_mtime <= mtime:
 742                 response = "x"
 743                 while response != "y" and response != "n":
 744                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746             if response == "y":
 747                 tmpFile = open(fileName, "rb")
 748                 message = tmpFile.read()
 749                 tmpFile.close()
 750                 submitTemplate = message[:message.index(separatorLine)]
 751                 if self.isWindows:
 752                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                 p4_write_pipe("submit -i", submitTemplate)
 754             else:
 755                 for f in editedFiles:
 756                     p4_system("revert \"%s\"" % f);
 757                 for f in filesToAdd:
 758                     p4_system("revert \"%s\"" % f);
 759                     system("rm %s" %f)
 760
 761             os.remove(fileName)
 762         else:
 763             fileName = "submit.txt"
 764             file = open(fileName, "w+")
 765             file.write(self.prepareLogMessage(template, logMessage))
 766             file.close()
 767             print ("Perforce submit template written as %s. "
 768                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                    % (fileName, fileName))
 770
 771     def run(self, args):
 772         if len(args) == 0:
 773             self.master = currentGitBranch()
 774             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                 die("Detecting current git branch failed!")
 776         elif len(args) == 1:
 777             self.master = args[0]
 778         else:
 779             return False
 780
 781         allowSubmit = gitConfig("git-p4.allowSubmit")
 782         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783             die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785         [upstream, settings] = findUpstreamBranchPoint()
 786         self.depotPath = settings['depot-paths'][0]
 787         if len(self.origin) == 0:
 788             self.origin = upstream
 789
 790         if self.verbose:
 791             print "Origin branch is " + self.origin
 792
 793         if len(self.depotPath) == 0:
 794             print "Internal error: cannot locate perforce depot path from existing branches"
 795             sys.exit(128)
 796
 797         self.clientPath = p4Where(self.depotPath)
 798
 799         if len(self.clientPath) == 0:
 800             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801             sys.exit(128)
 802
 803         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804         self.oldWorkingDirectory = os.getcwd()
 805
 806         chdir(self.clientPath)
 807         print "Syncronizing p4 checkout..."
 808         p4_system("sync ...")
 809
 810         self.check()
 811
 812         commits = []
 813         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814             commits.append(line.strip())
 815         commits.reverse()
 816
 817         while len(commits) > 0:
 818             commit = commits[0]
 819             commits = commits[1:]
 820             self.applyCommit(commit)
 821             if not self.interactive:
 822                 break
 823
 824         if len(commits) == 0:
 825             print "All changes applied!"
 826             chdir(self.oldWorkingDirectory)
 827
 828             sync = P4Sync()
 829             sync.run([])
 830
 831             rebase = P4Rebase()
 832             rebase.rebase()
 833
 834         return True
 835
 836 class P4Sync(Command):
 837     def __init__(self):
 838         Command.__init__(self)
 839         self.options = [
 840                 optparse.make_option("--branch", dest="branch"),
 841                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 842                 optparse.make_option("--changesfile", dest="changesFile"),
 843                 optparse.make_option("--silent", dest="silent", action="store_true"),
 844                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 845                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 846                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 847                                      help="Import into refs/heads/ , not refs/remotes"),
 848                 optparse.make_option("--max-changes", dest="maxChanges"),
 849                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 850                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 851                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 852                                      help="Only sync files that are included in the Perforce Client Spec")
 853         ]
 854         self.description = """Imports from Perforce into a git repository.\n
 855     example:
 856     //depot/my/project/ -- to import the current head
 857     //depot/my/project/@all -- to import everything
 858     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 859
 860     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 861
 862         self.usage += " //depot/path[@revRange]"
 863         self.silent = False
 864         self.createdBranches = Set()
 865         self.committedChanges = Set()
 866         self.branch = ""
 867         self.detectBranches = False
 868         self.detectLabels = False
 869         self.changesFile = ""
 870         self.syncWithOrigin = True
 871         self.verbose = False
 872         self.importIntoRemotes = True
 873         self.maxChanges = ""
 874         self.isWindows = (platform.system() == "Windows")
 875         self.keepRepoPath = False
 876         self.depotPaths = None
 877         self.p4BranchesInGit = []
 878         self.cloneExclude = []
 879         self.useClientSpec = False
 880         self.clientSpecDirs = []
 881
 882         if gitConfig("git-p4.syncFromOrigin") == "false":
 883             self.syncWithOrigin = False
 884
 885     def extractFilesFromCommit(self, commit):
 886         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 887                              for path in self.cloneExclude]
 888         files = []
 889         fnum = 0
 890         while commit.has_key("depotFile%s" % fnum):
 891             path =  commit["depotFile%s" % fnum]
 892
 893             if [p for p in self.cloneExclude
 894                 if path.startswith (p)]:
 895                 found = False
 896             else:
 897                 found = [p for p in self.depotPaths
 898                          if path.startswith (p)]
 899             if not found:
 900                 fnum = fnum + 1
 901                 continue
 902
 903             file = {}
 904             file["path"] = path
 905             file["rev"] = commit["rev%s" % fnum]
 906             file["action"] = commit["action%s" % fnum]
 907             file["type"] = commit["type%s" % fnum]
 908             files.append(file)
 909             fnum = fnum + 1
 910         return files
 911
 912     def stripRepoPath(self, path, prefixes):
 913         if self.keepRepoPath:
 914             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 915
 916         for p in prefixes:
 917             if path.startswith(p):
 918                 path = path[len(p):]
 919
 920         return path
 921
 922     def splitFilesIntoBranches(self, commit):
 923         branches = {}
 924         fnum = 0
 925         while commit.has_key("depotFile%s" % fnum):
 926             path =  commit["depotFile%s" % fnum]
 927             found = [p for p in self.depotPaths
 928                      if path.startswith (p)]
 929             if not found:
 930                 fnum = fnum + 1
 931                 continue
 932
 933             file = {}
 934             file["path"] = path
 935             file["rev"] = commit["rev%s" % fnum]
 936             file["action"] = commit["action%s" % fnum]
 937             file["type"] = commit["type%s" % fnum]
 938             fnum = fnum + 1
 939
 940             relPath = self.stripRepoPath(path, self.depotPaths)
 941
 942             for branch in self.knownBranches.keys():
 943
 944                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 945                 if relPath.startswith(branch + "/"):
 946                     if branch not in branches:
 947                         branches[branch] = []
 948                     branches[branch].append(file)
 949                     break
 950
 951         return branches
 952
 953     ## Should move this out, doesn't use SELF.
 954     def readP4Files(self, files):
 955         filesForCommit = []
 956         filesToRead = []
 957
 958         for f in files:
 959             includeFile = True
 960             for val in self.clientSpecDirs:
 961                 if f['path'].startswith(val[0]):
 962                     if val[1] <= 0:
 963                         includeFile = False
 964                     break
 965
 966             if includeFile:
 967                 filesForCommit.append(f)
 968                 if f['action'] not in ('delete', 'purge'):
 969                     filesToRead.append(f)
 970
 971         filedata = []
 972         if len(filesToRead) > 0:
 973             filedata = p4CmdList('-x - print',
 974                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 975                                                   for f in filesToRead]),
 976                                  stdin_mode='w+')
 977
 978             if "p4ExitCode" in filedata[0]:
 979                 die("Problems executing p4. Error: [%d]."
 980                     % (filedata[0]['p4ExitCode']));
 981
 982         j = 0;
 983         contents = {}
 984         while j < len(filedata):
 985             stat = filedata[j]
 986             j += 1
 987             text = ''
 988             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 989                 text += filedata[j]['data']
 990                 del filedata[j]['data']
 991                 j += 1
 992
 993             if not stat.has_key('depotFile'):
 994                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 995                 continue
 996
 997             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 998                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 999             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1000                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text)
1001
1002             contents[stat['depotFile']] = text
1003
1004         for f in filesForCommit:
1005             path = f['path']
1006             if contents.has_key(path):
1007                 f['data'] = contents[path]
1008
1009         return filesForCommit
1010
1011     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1012         epoch = details["time"]
1013         author = details["user"]
1014
1015         if self.verbose:
1016             print "commit into %s" % branch
1017
1018         # start with reading files; if that fails, we should not
1019         # create a commit.
1020         new_files = []
1021         for f in files:
1022             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1023                 new_files.append (f)
1024             else:
1025                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1026         files = self.readP4Files(new_files)
1027
1028         self.gitStream.write("commit %s\n" % branch)
1029 #        gitStream.write("mark :%s\n" % details["change"])
1030         self.committedChanges.add(int(details["change"]))
1031         committer = ""
1032         if author not in self.users:
1033             self.getUserMapFromPerforceServer()
1034         if author in self.users:
1035             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1036         else:
1037             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1038
1039         self.gitStream.write("committer %s\n" % committer)
1040
1041         self.gitStream.write("data <<EOT\n")
1042         self.gitStream.write(details["desc"])
1043         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1044                              % (','.join (branchPrefixes), details["change"]))
1045         if len(details['options']) > 0:
1046             self.gitStream.write(": options = %s" % details['options'])
1047         self.gitStream.write("]\nEOT\n\n")
1048
1049         if len(parent) > 0:
1050             if self.verbose:
1051                 print "parent %s" % parent
1052             self.gitStream.write("from %s\n" % parent)
1053
1054         for file in files:
1055             if file["type"] == "apple":
1056                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1057                 continue
1058
1059             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1060             if file["action"] in ("delete", "purge"):
1061                 self.gitStream.write("D %s\n" % relPath)
1062             else:
1063                 data = file['data']
1064
1065                 mode = "644"
1066                 if isP4Exec(file["type"]):
1067                     mode = "755"
1068                 elif file["type"] == "symlink":
1069                     mode = "120000"
1070                     # p4 print on a symlink contains "target\n", so strip it off
1071                     data = data[:-1]
1072
1073                 if self.isWindows and file["type"].endswith("text"):
1074                     data = data.replace("\r\n", "\n")
1075
1076                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1077                 self.gitStream.write("data %s\n" % len(data))
1078                 self.gitStream.write(data)
1079                 self.gitStream.write("\n")
1080
1081         self.gitStream.write("\n")
1082
1083         change = int(details["change"])
1084
1085         if self.labels.has_key(change):
1086             label = self.labels[change]
1087             labelDetails = label[0]
1088             labelRevisions = label[1]
1089             if self.verbose:
1090                 print "Change %s is labelled %s" % (change, labelDetails)
1091
1092             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1093                                                     for p in branchPrefixes]))
1094
1095             if len(files) == len(labelRevisions):
1096
1097                 cleanedFiles = {}
1098                 for info in files:
1099                     if info["action"] in ("delete", "purge"):
1100                         continue
1101                     cleanedFiles[info["depotFile"]] = info["rev"]
1102
1103                 if cleanedFiles == labelRevisions:
1104                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1105                     self.gitStream.write("from %s\n" % branch)
1106
1107                     owner = labelDetails["Owner"]
1108                     tagger = ""
1109                     if author in self.users:
1110                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1111                     else:
1112                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1113                     self.gitStream.write("tagger %s\n" % tagger)
1114                     self.gitStream.write("data <<EOT\n")
1115                     self.gitStream.write(labelDetails["Description"])
1116                     self.gitStream.write("EOT\n\n")
1117
1118                 else:
1119                     if not self.silent:
1120                         print ("Tag %s does not match with change %s: files do not match."
1121                                % (labelDetails["label"], change))
1122
1123             else:
1124                 if not self.silent:
1125                     print ("Tag %s does not match with change %s: file count is different."
1126                            % (labelDetails["label"], change))
1127
1128     def getUserCacheFilename(self):
1129         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1130         return home + "/.gitp4-usercache.txt"
1131
1132     def getUserMapFromPerforceServer(self):
1133         if self.userMapFromPerforceServer:
1134             return
1135         self.users = {}
1136
1137         for output in p4CmdList("users"):
1138             if not output.has_key("User"):
1139                 continue
1140             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1141
1142
1143         s = ''
1144         for (key, val) in self.users.items():
1145             s += "%s\t%s\n" % (key, val)
1146
1147         open(self.getUserCacheFilename(), "wb").write(s)
1148         self.userMapFromPerforceServer = True
1149
1150     def loadUserMapFromCache(self):
1151         self.users = {}
1152         self.userMapFromPerforceServer = False
1153         try:
1154             cache = open(self.getUserCacheFilename(), "rb")
1155             lines = cache.readlines()
1156             cache.close()
1157             for line in lines:
1158                 entry = line.strip().split("\t")
1159                 self.users[entry[0]] = entry[1]
1160         except IOError:
1161             self.getUserMapFromPerforceServer()
1162
1163     def getLabels(self):
1164         self.labels = {}
1165
1166         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1167         if len(l) > 0 and not self.silent:
1168             print "Finding files belonging to labels in %s" % `self.depotPaths`
1169
1170         for output in l:
1171             label = output["label"]
1172             revisions = {}
1173             newestChange = 0
1174             if self.verbose:
1175                 print "Querying files for label %s" % label
1176             for file in p4CmdList("files "
1177                                   +  ' '.join (["%s...@%s" % (p, label)
1178                                                 for p in self.depotPaths])):
1179                 revisions[file["depotFile"]] = file["rev"]
1180                 change = int(file["change"])
1181                 if change > newestChange:
1182                     newestChange = change
1183
1184             self.labels[newestChange] = [output, revisions]
1185
1186         if self.verbose:
1187             print "Label changes: %s" % self.labels.keys()
1188
1189     def guessProjectName(self):
1190         for p in self.depotPaths:
1191             if p.endswith("/"):
1192                 p = p[:-1]
1193             p = p[p.strip().rfind("/") + 1:]
1194             if not p.endswith("/"):
1195                p += "/"
1196             return p
1197
1198     def getBranchMapping(self):
1199         lostAndFoundBranches = set()
1200
1201         for info in p4CmdList("branches"):
1202             details = p4Cmd("branch -o %s" % info["branch"])
1203             viewIdx = 0
1204             while details.has_key("View%s" % viewIdx):
1205                 paths = details["View%s" % viewIdx].split(" ")
1206                 viewIdx = viewIdx + 1
1207                 # require standard //depot/foo/... //depot/bar/... mapping
1208                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1209                     continue
1210                 source = paths[0]
1211                 destination = paths[1]
1212                 ## HACK
1213                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1214                     source = source[len(self.depotPaths[0]):-4]
1215                     destination = destination[len(self.depotPaths[0]):-4]
1216
1217                     if destination in self.knownBranches:
1218                         if not self.silent:
1219                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1220                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1221                         continue
1222
1223                     self.knownBranches[destination] = source
1224
1225                     lostAndFoundBranches.discard(destination)
1226
1227                     if source not in self.knownBranches:
1228                         lostAndFoundBranches.add(source)
1229
1230
1231         for branch in lostAndFoundBranches:
1232             self.knownBranches[branch] = branch
1233
1234     def getBranchMappingFromGitBranches(self):
1235         branches = p4BranchesInGit(self.importIntoRemotes)
1236         for branch in branches.keys():
1237             if branch == "master":
1238                 branch = "main"
1239             else:
1240                 branch = branch[len(self.projectName):]
1241             self.knownBranches[branch] = branch
1242
1243     def listExistingP4GitBranches(self):
1244         # branches holds mapping from name to commit
1245         branches = p4BranchesInGit(self.importIntoRemotes)
1246         self.p4BranchesInGit = branches.keys()
1247         for branch in branches.keys():
1248             self.initialParents[self.refPrefix + branch] = branches[branch]
1249
1250     def updateOptionDict(self, d):
1251         option_keys = {}
1252         if self.keepRepoPath:
1253             option_keys['keepRepoPath'] = 1
1254
1255         d["options"] = ' '.join(sorted(option_keys.keys()))
1256
1257     def readOptions(self, d):
1258         self.keepRepoPath = (d.has_key('options')
1259                              and ('keepRepoPath' in d['options']))
1260
1261     def gitRefForBranch(self, branch):
1262         if branch == "main":
1263             return self.refPrefix + "master"
1264
1265         if len(branch) <= 0:
1266             return branch
1267
1268         return self.refPrefix + self.projectName + branch
1269
1270     def gitCommitByP4Change(self, ref, change):
1271         if self.verbose:
1272             print "looking in ref " + ref + " for change %s using bisect..." % change
1273
1274         earliestCommit = ""
1275         latestCommit = parseRevision(ref)
1276
1277         while True:
1278             if self.verbose:
1279                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1280             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1281             if len(next) == 0:
1282                 if self.verbose:
1283                     print "argh"
1284                 return ""
1285             log = extractLogMessageFromGitCommit(next)
1286             settings = extractSettingsGitLog(log)
1287             currentChange = int(settings['change'])
1288             if self.verbose:
1289                 print "current change %s" % currentChange
1290
1291             if currentChange == change:
1292                 if self.verbose:
1293                     print "found %s" % next
1294                 return next
1295
1296             if currentChange < change:
1297                 earliestCommit = "^%s" % next
1298             else:
1299                 latestCommit = "%s" % next
1300
1301         return ""
1302
1303     def importNewBranch(self, branch, maxChange):
1304         # make fast-import flush all changes to disk and update the refs using the checkpoint
1305         # command so that we can try to find the branch parent in the git history
1306         self.gitStream.write("checkpoint\n\n");
1307         self.gitStream.flush();
1308         branchPrefix = self.depotPaths[0] + branch + "/"
1309         range = "@1,%s" % maxChange
1310         #print "prefix" + branchPrefix
1311         changes = p4ChangesForPaths([branchPrefix], range)
1312         if len(changes) <= 0:
1313             return False
1314         firstChange = changes[0]
1315         #print "first change in branch: %s" % firstChange
1316         sourceBranch = self.knownBranches[branch]
1317         sourceDepotPath = self.depotPaths[0] + sourceBranch
1318         sourceRef = self.gitRefForBranch(sourceBranch)
1319         #print "source " + sourceBranch
1320
1321         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1322         #print "branch parent: %s" % branchParentChange
1323         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1324         if len(gitParent) > 0:
1325             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1326             #print "parent git commit: %s" % gitParent
1327
1328         self.importChanges(changes)
1329         return True
1330
1331     def importChanges(self, changes):
1332         cnt = 1
1333         for change in changes:
1334             description = p4Cmd("describe %s" % change)
1335             self.updateOptionDict(description)
1336
1337             if not self.silent:
1338                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1339                 sys.stdout.flush()
1340             cnt = cnt + 1
1341
1342             try:
1343                 if self.detectBranches:
1344                     branches = self.splitFilesIntoBranches(description)
1345                     for branch in branches.keys():
1346                         ## HACK  --hwn
1347                         branchPrefix = self.depotPaths[0] + branch + "/"
1348
1349                         parent = ""
1350
1351                         filesForCommit = branches[branch]
1352
1353                         if self.verbose:
1354                             print "branch is %s" % branch
1355
1356                         self.updatedBranches.add(branch)
1357
1358                         if branch not in self.createdBranches:
1359                             self.createdBranches.add(branch)
1360                             parent = self.knownBranches[branch]
1361                             if parent == branch:
1362                                 parent = ""
1363                             else:
1364                                 fullBranch = self.projectName + branch
1365                                 if fullBranch not in self.p4BranchesInGit:
1366                                     if not self.silent:
1367                                         print("\n    Importing new branch %s" % fullBranch);
1368                                     if self.importNewBranch(branch, change - 1):
1369                                         parent = ""
1370                                         self.p4BranchesInGit.append(fullBranch)
1371                                     if not self.silent:
1372                                         print("\n    Resuming with change %s" % change);
1373
1374                                 if self.verbose:
1375                                     print "parent determined through known branches: %s" % parent
1376
1377                         branch = self.gitRefForBranch(branch)
1378                         parent = self.gitRefForBranch(parent)
1379
1380                         if self.verbose:
1381                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1382
1383                         if len(parent) == 0 and branch in self.initialParents:
1384                             parent = self.initialParents[branch]
1385                             del self.initialParents[branch]
1386
1387                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1388                 else:
1389                     files = self.extractFilesFromCommit(description)
1390                     self.commit(description, files, self.branch, self.depotPaths,
1391                                 self.initialParent)
1392                     self.initialParent = ""
1393             except IOError:
1394                 print self.gitError.read()
1395                 sys.exit(1)
1396
1397     def importHeadRevision(self, revision):
1398         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1399
1400         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1401         details["desc"] = ("Initial import of %s from the state at revision %s"
1402                            % (' '.join(self.depotPaths), revision))
1403         details["change"] = revision
1404         newestRevision = 0
1405
1406         fileCnt = 0
1407         for info in p4CmdList("files "
1408                               +  ' '.join(["%s...%s"
1409                                            % (p, revision)
1410                                            for p in self.depotPaths])):
1411
1412             if info['code'] == 'error':
1413                 sys.stderr.write("p4 returned an error: %s\n"
1414                                  % info['data'])
1415                 sys.exit(1)
1416
1417
1418             change = int(info["change"])
1419             if change > newestRevision:
1420                 newestRevision = change
1421
1422             if info["action"] in ("delete", "purge"):
1423                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1424                 #fileCnt = fileCnt + 1
1425                 continue
1426
1427             for prop in ["depotFile", "rev", "action", "type" ]:
1428                 details["%s%s" % (prop, fileCnt)] = info[prop]
1429
1430             fileCnt = fileCnt + 1
1431
1432         details["change"] = newestRevision
1433         self.updateOptionDict(details)
1434         try:
1435             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1436         except IOError:
1437             print "IO error with git fast-import. Is your git version recent enough?"
1438             print self.gitError.read()
1439
1440
1441     def getClientSpec(self):
1442         specList = p4CmdList( "client -o" )
1443         temp = {}
1444         for entry in specList:
1445             for k,v in entry.iteritems():
1446                 if k.startswith("View"):
1447                     if v.startswith('"'):
1448                         start = 1
1449                     else:
1450                         start = 0
1451                     index = v.find("...")
1452                     v = v[start:index]
1453                     if v.startswith("-"):
1454                         v = v[1:]
1455                         temp[v] = -len(v)
1456                     else:
1457                         temp[v] = len(v)
1458         self.clientSpecDirs = temp.items()
1459         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1460
1461     def run(self, args):
1462         self.depotPaths = []
1463         self.changeRange = ""
1464         self.initialParent = ""
1465         self.previousDepotPaths = []
1466
1467         # map from branch depot path to parent branch
1468         self.knownBranches = {}
1469         self.initialParents = {}
1470         self.hasOrigin = originP4BranchesExist()
1471         if not self.syncWithOrigin:
1472             self.hasOrigin = False
1473
1474         if self.importIntoRemotes:
1475             self.refPrefix = "refs/remotes/p4/"
1476         else:
1477             self.refPrefix = "refs/heads/p4/"
1478
1479         if self.syncWithOrigin and self.hasOrigin:
1480             if not self.silent:
1481                 print "Syncing with origin first by calling git fetch origin"
1482             system("git fetch origin")
1483
1484         if len(self.branch) == 0:
1485             self.branch = self.refPrefix + "master"
1486             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1487                 system("git update-ref %s refs/heads/p4" % self.branch)
1488                 system("git branch -D p4");
1489             # create it /after/ importing, when master exists
1490             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1491                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1492
1493         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1494             self.getClientSpec()
1495
1496         # TODO: should always look at previous commits,
1497         # merge with previous imports, if possible.
1498         if args == []:
1499             if self.hasOrigin:
1500                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1501             self.listExistingP4GitBranches()
1502
1503             if len(self.p4BranchesInGit) > 1:
1504                 if not self.silent:
1505                     print "Importing from/into multiple branches"
1506                 self.detectBranches = True
1507
1508             if self.verbose:
1509                 print "branches: %s" % self.p4BranchesInGit
1510
1511             p4Change = 0
1512             for branch in self.p4BranchesInGit:
1513                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1514
1515                 settings = extractSettingsGitLog(logMsg)
1516
1517                 self.readOptions(settings)
1518                 if (settings.has_key('depot-paths')
1519                     and settings.has_key ('change')):
1520                     change = int(settings['change']) + 1
1521                     p4Change = max(p4Change, change)
1522
1523                     depotPaths = sorted(settings['depot-paths'])
1524                     if self.previousDepotPaths == []:
1525                         self.previousDepotPaths = depotPaths
1526                     else:
1527                         paths = []
1528                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1529                             for i in range(0, min(len(cur), len(prev))):
1530                                 if cur[i] <> prev[i]:
1531                                     i = i - 1
1532                                     break
1533
1534                             paths.append (cur[:i + 1])
1535
1536                         self.previousDepotPaths = paths
1537
1538             if p4Change > 0:
1539                 self.depotPaths = sorted(self.previousDepotPaths)
1540                 self.changeRange = "@%s,#head" % p4Change
1541                 if not self.detectBranches:
1542                     self.initialParent = parseRevision(self.branch)
1543                 if not self.silent and not self.detectBranches:
1544                     print "Performing incremental import into %s git branch" % self.branch
1545
1546         if not self.branch.startswith("refs/"):
1547             self.branch = "refs/heads/" + self.branch
1548
1549         if len(args) == 0 and self.depotPaths:
1550             if not self.silent:
1551                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1552         else:
1553             if self.depotPaths and self.depotPaths != args:
1554                 print ("previous import used depot path %s and now %s was specified. "
1555                        "This doesn't work!" % (' '.join (self.depotPaths),
1556                                                ' '.join (args)))
1557                 sys.exit(1)
1558
1559             self.depotPaths = sorted(args)
1560
1561         revision = ""
1562         self.users = {}
1563
1564         newPaths = []
1565         for p in self.depotPaths:
1566             if p.find("@") != -1:
1567                 atIdx = p.index("@")
1568                 self.changeRange = p[atIdx:]
1569                 if self.changeRange == "@all":
1570                     self.changeRange = ""
1571                 elif ',' not in self.changeRange:
1572                     revision = self.changeRange
1573                     self.changeRange = ""
1574                 p = p[:atIdx]
1575             elif p.find("#") != -1:
1576                 hashIdx = p.index("#")
1577                 revision = p[hashIdx:]
1578                 p = p[:hashIdx]
1579             elif self.previousDepotPaths == []:
1580                 revision = "#head"
1581
1582             p = re.sub ("\.\.\.$", "", p)
1583             if not p.endswith("/"):
1584                 p += "/"
1585
1586             newPaths.append(p)
1587
1588         self.depotPaths = newPaths
1589
1590
1591         self.loadUserMapFromCache()
1592         self.labels = {}
1593         if self.detectLabels:
1594             self.getLabels();
1595
1596         if self.detectBranches:
1597             ## FIXME - what's a P4 projectName ?
1598             self.projectName = self.guessProjectName()
1599
1600             if self.hasOrigin:
1601                 self.getBranchMappingFromGitBranches()
1602             else:
1603                 self.getBranchMapping()
1604             if self.verbose:
1605                 print "p4-git branches: %s" % self.p4BranchesInGit
1606                 print "initial parents: %s" % self.initialParents
1607             for b in self.p4BranchesInGit:
1608                 if b != "master":
1609
1610                     ## FIXME
1611                     b = b[len(self.projectName):]
1612                 self.createdBranches.add(b)
1613
1614         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1615
1616         importProcess = subprocess.Popen(["git", "fast-import"],
1617                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1618                                          stderr=subprocess.PIPE);
1619         self.gitOutput = importProcess.stdout
1620         self.gitStream = importProcess.stdin
1621         self.gitError = importProcess.stderr
1622
1623         if revision:
1624             self.importHeadRevision(revision)
1625         else:
1626             changes = []
1627
1628             if len(self.changesFile) > 0:
1629                 output = open(self.changesFile).readlines()
1630                 changeSet = Set()
1631                 for line in output:
1632                     changeSet.add(int(line))
1633
1634                 for change in changeSet:
1635                     changes.append(change)
1636
1637                 changes.sort()
1638             else:
1639                 if self.verbose:
1640                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1641                                                               self.changeRange)
1642                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1643
1644                 if len(self.maxChanges) > 0:
1645                     changes = changes[:min(int(self.maxChanges), len(changes))]
1646
1647             if len(changes) == 0:
1648                 if not self.silent:
1649                     print "No changes to import!"
1650                 return True
1651
1652             if not self.silent and not self.detectBranches:
1653                 print "Import destination: %s" % self.branch
1654
1655             self.updatedBranches = set()
1656
1657             self.importChanges(changes)
1658
1659             if not self.silent:
1660                 print ""
1661                 if len(self.updatedBranches) > 0:
1662                     sys.stdout.write("Updated branches: ")
1663                     for b in self.updatedBranches:
1664                         sys.stdout.write("%s " % b)
1665                     sys.stdout.write("\n")
1666
1667         self.gitStream.close()
1668         if importProcess.wait() != 0:
1669             die("fast-import failed: %s" % self.gitError.read())
1670         self.gitOutput.close()
1671         self.gitError.close()
1672
1673         return True
1674
1675 class P4Rebase(Command):
1676     def __init__(self):
1677         Command.__init__(self)
1678         self.options = [ ]
1679         self.description = ("Fetches the latest revision from perforce and "
1680                             + "rebases the current work (branch) against it")
1681         self.verbose = False
1682
1683     def run(self, args):
1684         sync = P4Sync()
1685         sync.run([])
1686
1687         return self.rebase()
1688
1689     def rebase(self):
1690         if os.system("git update-index --refresh") != 0:
1691             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1692         if len(read_pipe("git diff-index HEAD --")) > 0:
1693             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1694
1695         [upstream, settings] = findUpstreamBranchPoint()
1696         if len(upstream) == 0:
1697             die("Cannot find upstream branchpoint for rebase")
1698
1699         # the branchpoint may be p4/foo~3, so strip off the parent
1700         upstream = re.sub("~[0-9]+$", "", upstream)
1701
1702         print "Rebasing the current branch onto %s" % upstream
1703         oldHead = read_pipe("git rev-parse HEAD").strip()
1704         system("git rebase %s" % upstream)
1705         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1706         return True
1707
1708 class P4Clone(P4Sync):
1709     def __init__(self):
1710         P4Sync.__init__(self)
1711         self.description = "Creates a new git repository and imports from Perforce into it"
1712         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1713         self.options += [
1714             optparse.make_option("--destination", dest="cloneDestination",
1715                                  action='store', default=None,
1716                                  help="where to leave result of the clone"),
1717             optparse.make_option("-/", dest="cloneExclude",
1718                                  action="append", type="string",
1719                                  help="exclude depot path")
1720         ]
1721         self.cloneDestination = None
1722         self.needsGit = False
1723
1724     # This is required for the "append" cloneExclude action
1725     def ensure_value(self, attr, value):
1726         if not hasattr(self, attr) or getattr(self, attr) is None:
1727             setattr(self, attr, value)
1728         return getattr(self, attr)
1729
1730     def defaultDestination(self, args):
1731         ## TODO: use common prefix of args?
1732         depotPath = args[0]
1733         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1734         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1735         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1736         depotDir = re.sub(r"/$", "", depotDir)
1737         return os.path.split(depotDir)[1]
1738
1739     def run(self, args):
1740         if len(args) < 1:
1741             return False
1742
1743         if self.keepRepoPath and not self.cloneDestination:
1744             sys.stderr.write("Must specify destination for --keep-path\n")
1745             sys.exit(1)
1746
1747         depotPaths = args
1748
1749         if not self.cloneDestination and len(depotPaths) > 1:
1750             self.cloneDestination = depotPaths[-1]
1751             depotPaths = depotPaths[:-1]
1752
1753         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1754         for p in depotPaths:
1755             if not p.startswith("//"):
1756                 return False
1757
1758         if not self.cloneDestination:
1759             self.cloneDestination = self.defaultDestination(args)
1760
1761         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1762         if not os.path.exists(self.cloneDestination):
1763             os.makedirs(self.cloneDestination)
1764         chdir(self.cloneDestination)
1765         system("git init")
1766         self.gitdir = os.getcwd() + "/.git"
1767         if not P4Sync.run(self, depotPaths):
1768             return False
1769         if self.branch != "master":
1770             if self.importIntoRemotes:
1771                 masterbranch = "refs/remotes/p4/master"
1772             else:
1773                 masterbranch = "refs/heads/p4/master"
1774             if gitBranchExists(masterbranch):
1775                 system("git branch master %s" % masterbranch)
1776                 system("git checkout -f")
1777             else:
1778                 print "Could not detect main branch. No checkout/master branch created."
1779
1780         return True
1781
1782 class P4Branches(Command):
1783     def __init__(self):
1784         Command.__init__(self)
1785         self.options = [ ]
1786         self.description = ("Shows the git branches that hold imports and their "
1787                             + "corresponding perforce depot paths")
1788         self.verbose = False
1789
1790     def run(self, args):
1791         if originP4BranchesExist():
1792             createOrUpdateBranchesFromOrigin()
1793
1794         cmdline = "git rev-parse --symbolic "
1795         cmdline += " --remotes"
1796
1797         for line in read_pipe_lines(cmdline):
1798             line = line.strip()
1799
1800             if not line.startswith('p4/') or line == "p4/HEAD":
1801                 continue
1802             branch = line
1803
1804             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1805             settings = extractSettingsGitLog(log)
1806
1807             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1808         return True
1809
1810 class HelpFormatter(optparse.IndentedHelpFormatter):
1811     def __init__(self):
1812         optparse.IndentedHelpFormatter.__init__(self)
1813
1814     def format_description(self, description):
1815         if description:
1816             return description + "\n"
1817         else:
1818             return ""
1819
1820 def printUsage(commands):
1821     print "usage: %s <command> [options]" % sys.argv[0]
1822     print ""
1823     print "valid commands: %s" % ", ".join(commands)
1824     print ""
1825     print "Try %s <command> --help for command specific help." % sys.argv[0]
1826     print ""
1827
1828 commands = {
1829     "debug" : P4Debug,
1830     "submit" : P4Submit,
1831     "commit" : P4Submit,
1832     "sync" : P4Sync,
1833     "rebase" : P4Rebase,
1834     "clone" : P4Clone,
1835     "rollback" : P4RollBack,
1836     "branches" : P4Branches
1837 }
1838
1839
1840 def main():
1841     if len(sys.argv[1:]) == 0:
1842         printUsage(commands.keys())
1843         sys.exit(2)
1844
1845     cmd = ""
1846     cmdName = sys.argv[1]
1847     try:
1848         klass = commands[cmdName]
1849         cmd = klass()
1850     except KeyError:
1851         print "unknown command %s" % cmdName
1852         print ""
1853         printUsage(commands.keys())
1854         sys.exit(2)
1855
1856     options = cmd.options
1857     cmd.gitdir = os.environ.get("GIT_DIR", None)
1858
1859     args = sys.argv[2:]
1860
1861     if len(options) > 0:
1862         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1863
1864         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1865                                        options,
1866                                        description = cmd.description,
1867                                        formatter = HelpFormatter())
1868
1869         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1870     global verbose
1871     verbose = cmd.verbose
1872     if cmd.needsGit:
1873         if cmd.gitdir == None:
1874             cmd.gitdir = os.path.abspath(".git")
1875             if not isValidGitDir(cmd.gitdir):
1876                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1877                 if os.path.exists(cmd.gitdir):
1878                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1879                     if len(cdup) > 0:
1880                         chdir(cdup);
1881
1882         if not isValidGitDir(cmd.gitdir):
1883             if isValidGitDir(cmd.gitdir + "/.git"):
1884                 cmd.gitdir += "/.git"
1885             else:
1886                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1887
1888         os.environ["GIT_DIR"] = cmd.gitdir
1889
1890     if not cmd.run(args):
1891         parser.print_help()
1892
1893
1894 if __name__ == '__main__':
1895     main()