contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s" % file)
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                 optparse.make_option("--origin", dest="origin"),
 470                 optparse.make_option("--reset", action="store_true", dest="reset"),
 471                 optparse.make_option("--log-substitutions", dest="substFile"),
 472                 optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 473                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 474         ]
 475         self.description = "Submit changes from git to the perforce depot."
 476         self.usage += " [name of git branch to submit into perforce depot]"
 477         self.firstTime = True
 478         self.reset = False
 479         self.interactive = True
 480         self.substFile = ""
 481         self.firstTime = True
 482         self.origin = ""
 483         self.directSubmit = False
 484         self.detectRename = False
 485         self.verbose = False
 486         self.isWindows = (platform.system() == "Windows")
 487
 488         self.logSubstitutions = {}
 489         self.logSubstitutions["<enter description here>"] = "%log%"
 490         self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 491
 492     def check(self):
 493         if len(p4CmdList("opened ...")) > 0:
 494             die("You have files opened with perforce! Close them before starting the sync.")
 495
 496     def start(self):
 497         if len(self.config) > 0 and not self.reset:
 498             die("Cannot start sync. Previous sync config found at %s\n"
 499                 "If you want to start submitting again from scratch "
 500                 "maybe you want to call git-p4 submit --reset" % self.configFile)
 501
 502         commits = []
 503         if self.directSubmit:
 504             commits.append("0")
 505         else:
 506             for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 507                 commits.append(line.strip())
 508             commits.reverse()
 509
 510         self.config["commits"] = commits
 511
 512     def prepareLogMessage(self, template, message):
 513         result = ""
 514
 515         for line in template.split("\n"):
 516             if line.startswith("#"):
 517                 result += line + "\n"
 518                 continue
 519
 520             substituted = False
 521             for key in self.logSubstitutions.keys():
 522                 if line.find(key) != -1:
 523                     value = self.logSubstitutions[key]
 524                     value = value.replace("%log%", message)
 525                     if value != "@remove@":
 526                         result += line.replace(key, value) + "\n"
 527                     substituted = True
 528                     break
 529
 530             if not substituted:
 531                 result += line + "\n"
 532
 533         return result
 534
 535     def prepareSubmitTemplate(self):
 536         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 537         template = ""
 538         inFilesSection = False
 539         for line in read_pipe_lines("p4 change -o"):
 540             if inFilesSection:
 541                 if line.startswith("\t"):
 542                     # path starts and ends with a tab
 543                     path = line[1:]
 544                     lastTab = path.rfind("\t")
 545                     if lastTab != -1:
 546                         path = path[:lastTab]
 547                         if not path.startswith(self.depotPath):
 548                             continue
 549                 else:
 550                     inFilesSection = False
 551             else:
 552                 if line.startswith("Files:"):
 553                     inFilesSection = True
 554
 555             template += line
 556
 557         return template
 558
 559     def applyCommit(self, id):
 560         if self.directSubmit:
 561             print "Applying local change in working directory/index"
 562             diff = self.diffStatus
 563         else:
 564             print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 565             diffOpts = ("", "-M")[self.detectRename]
 566             diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 567         filesToAdd = set()
 568         filesToDelete = set()
 569         editedFiles = set()
 570         filesToChangeExecBit = {}
 571         for line in diff:
 572             diff = parseDiffTreeEntry(line)
 573             modifier = diff['status']
 574             path = diff['src']
 575             if modifier == "M":
 576                 system("p4 edit \"%s\"" % path)
 577                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 578                     filesToChangeExecBit[path] = diff['dst_mode']
 579                 editedFiles.add(path)
 580             elif modifier == "A":
 581                 filesToAdd.add(path)
 582                 filesToChangeExecBit[path] = diff['dst_mode']
 583                 if path in filesToDelete:
 584                     filesToDelete.remove(path)
 585             elif modifier == "D":
 586                 filesToDelete.add(path)
 587                 if path in filesToAdd:
 588                     filesToAdd.remove(path)
 589             elif modifier == "R":
 590                 src, dest = diff['src'], diff['dst']
 591                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 592                 system("p4 edit \"%s\"" % (dest))
 593                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 594                     filesToChangeExecBit[dest] = diff['dst_mode']
 595                 os.unlink(dest)
 596                 editedFiles.add(dest)
 597                 filesToDelete.add(src)
 598             else:
 599                 die("unknown modifier %s for %s" % (modifier, path))
 600
 601         if self.directSubmit:
 602             diffcmd = "cat \"%s\"" % self.diffFile
 603         else:
 604             diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 605         patchcmd = diffcmd + " | git apply "
 606         tryPatchCmd = patchcmd + "--check -"
 607         applyPatchCmd = patchcmd + "--check --apply -"
 608
 609         if os.system(tryPatchCmd) != 0:
 610             print "Unfortunately applying the change failed!"
 611             print "What do you want to do?"
 612             response = "x"
 613             while response != "s" and response != "a" and response != "w":
 614                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 615                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 616             if response == "s":
 617                 print "Skipping! Good luck with the next patches..."
 618                 for f in editedFiles:
 619                     system("p4 revert \"%s\"" % f);
 620                 for f in filesToAdd:
 621                     system("rm %s" %f)
 622                 return
 623             elif response == "a":
 624                 os.system(applyPatchCmd)
 625                 if len(filesToAdd) > 0:
 626                     print "You may also want to call p4 add on the following files:"
 627                     print " ".join(filesToAdd)
 628                 if len(filesToDelete):
 629                     print "The following files should be scheduled for deletion with p4 delete:"
 630                     print " ".join(filesToDelete)
 631                 die("Please resolve and submit the conflict manually and "
 632                     + "continue afterwards with git-p4 submit --continue")
 633             elif response == "w":
 634                 system(diffcmd + " > patch.txt")
 635                 print "Patch saved to patch.txt in %s !" % self.clientPath
 636                 die("Please resolve and submit the conflict manually and "
 637                     "continue afterwards with git-p4 submit --continue")
 638
 639         system(applyPatchCmd)
 640
 641         for f in filesToAdd:
 642             system("p4 add \"%s\"" % f)
 643         for f in filesToDelete:
 644             system("p4 revert \"%s\"" % f)
 645             system("p4 delete \"%s\"" % f)
 646
 647         # Set/clear executable bits
 648         for f in filesToChangeExecBit.keys():
 649             mode = filesToChangeExecBit[f]
 650             setP4ExecBit(f, mode)
 651
 652         logMessage = ""
 653         if not self.directSubmit:
 654             logMessage = extractLogMessageFromGitCommit(id)
 655             logMessage = logMessage.replace("\n", "\n\t")
 656             if self.isWindows:
 657                 logMessage = logMessage.replace("\n", "\r\n")
 658             logMessage = logMessage.strip()
 659
 660         template = self.prepareSubmitTemplate()
 661
 662         if self.interactive:
 663             submitTemplate = self.prepareLogMessage(template, logMessage)
 664             diff = read_pipe("p4 diff -du ...")
 665
 666             for newFile in filesToAdd:
 667                 diff += "==== new file ====\n"
 668                 diff += "--- /dev/null\n"
 669                 diff += "+++ %s\n" % newFile
 670                 f = open(newFile, "r")
 671                 for line in f.readlines():
 672                     diff += "+" + line
 673                 f.close()
 674
 675             separatorLine = "######## everything below this line is just the diff #######"
 676             if platform.system() == "Windows":
 677                 separatorLine += "\r"
 678             separatorLine += "\n"
 679
 680             [handle, fileName] = tempfile.mkstemp()
 681             tmpFile = os.fdopen(handle, "w+")
 682             tmpFile.write(submitTemplate + separatorLine + diff)
 683             tmpFile.close()
 684             defaultEditor = "vi"
 685             if platform.system() == "Windows":
 686                 defaultEditor = "notepad"
 687             editor = os.environ.get("EDITOR", defaultEditor);
 688             system(editor + " " + fileName)
 689             tmpFile = open(fileName, "rb")
 690             message = tmpFile.read()
 691             tmpFile.close()
 692             os.remove(fileName)
 693             submitTemplate = message[:message.index(separatorLine)]
 694             if self.isWindows:
 695                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 696
 697             if self.directSubmit:
 698                 print "Submitting to git first"
 699                 os.chdir(self.oldWorkingDirectory)
 700                 write_pipe("git commit -a -F -", submitTemplate)
 701                 os.chdir(self.clientPath)
 702
 703             write_pipe("p4 submit -i", submitTemplate)
 704         else:
 705             fileName = "submit.txt"
 706             file = open(fileName, "w+")
 707             file.write(self.prepareLogMessage(template, logMessage))
 708             file.close()
 709             print ("Perforce submit template written as %s. "
 710                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 711                    % (fileName, fileName))
 712
 713     def run(self, args):
 714         if len(args) == 0:
 715             self.master = currentGitBranch()
 716             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 717                 die("Detecting current git branch failed!")
 718         elif len(args) == 1:
 719             self.master = args[0]
 720         else:
 721             return False
 722
 723         [upstream, settings] = findUpstreamBranchPoint()
 724         self.depotPath = settings['depot-paths'][0]
 725         if len(self.origin) == 0:
 726             self.origin = upstream
 727
 728         if self.verbose:
 729             print "Origin branch is " + self.origin
 730
 731         if len(self.depotPath) == 0:
 732             print "Internal error: cannot locate perforce depot path from existing branches"
 733             sys.exit(128)
 734
 735         self.clientPath = p4Where(self.depotPath)
 736
 737         if len(self.clientPath) == 0:
 738             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 739             sys.exit(128)
 740
 741         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 742         self.oldWorkingDirectory = os.getcwd()
 743
 744         if self.directSubmit:
 745             self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 746             if len(self.diffStatus) == 0:
 747                 print "No changes in working directory to submit."
 748                 return True
 749             patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 750             self.diffFile = self.gitdir + "/p4-git-diff"
 751             f = open(self.diffFile, "wb")
 752             f.write(patch)
 753             f.close();
 754
 755         os.chdir(self.clientPath)
 756         print "Syncronizing p4 checkout..."
 757         system("p4 sync ...")
 758
 759         if self.reset:
 760             self.firstTime = True
 761
 762         if len(self.substFile) > 0:
 763             for line in open(self.substFile, "r").readlines():
 764                 tokens = line.strip().split("=")
 765                 self.logSubstitutions[tokens[0]] = tokens[1]
 766
 767         self.check()
 768         self.configFile = self.gitdir + "/p4-git-sync.cfg"
 769         self.config = shelve.open(self.configFile, writeback=True)
 770
 771         if self.firstTime:
 772             self.start()
 773
 774         commits = self.config.get("commits", [])
 775
 776         while len(commits) > 0:
 777             self.firstTime = False
 778             commit = commits[0]
 779             commits = commits[1:]
 780             self.config["commits"] = commits
 781             self.applyCommit(commit)
 782             if not self.interactive:
 783                 break
 784
 785         self.config.close()
 786
 787         if self.directSubmit:
 788             os.remove(self.diffFile)
 789
 790         if len(commits) == 0:
 791             if self.firstTime:
 792                 print "No changes found to apply between %s and current HEAD" % self.origin
 793             else:
 794                 print "All changes applied!"
 795                 os.chdir(self.oldWorkingDirectory)
 796
 797                 sync = P4Sync()
 798                 sync.run([])
 799
 800                 rebase = P4Rebase()
 801                 rebase.rebase()
 802             os.remove(self.configFile)
 803
 804         return True
 805
 806 class P4Sync(Command):
 807     def __init__(self):
 808         Command.__init__(self)
 809         self.options = [
 810                 optparse.make_option("--branch", dest="branch"),
 811                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 812                 optparse.make_option("--changesfile", dest="changesFile"),
 813                 optparse.make_option("--silent", dest="silent", action="store_true"),
 814                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 815                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 816                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 817                                      help="Import into refs/heads/ , not refs/remotes"),
 818                 optparse.make_option("--max-changes", dest="maxChanges"),
 819                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 820                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 821         ]
 822         self.description = """Imports from Perforce into a git repository.\n
 823     example:
 824     //depot/my/project/ -- to import the current head
 825     //depot/my/project/@all -- to import everything
 826     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 827
 828     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 829
 830         self.usage += " //depot/path[@revRange]"
 831         self.silent = False
 832         self.createdBranches = Set()
 833         self.committedChanges = Set()
 834         self.branch = ""
 835         self.detectBranches = False
 836         self.detectLabels = False
 837         self.changesFile = ""
 838         self.syncWithOrigin = True
 839         self.verbose = False
 840         self.importIntoRemotes = True
 841         self.maxChanges = ""
 842         self.isWindows = (platform.system() == "Windows")
 843         self.keepRepoPath = False
 844         self.depotPaths = None
 845         self.p4BranchesInGit = []
 846
 847         if gitConfig("git-p4.syncFromOrigin") == "false":
 848             self.syncWithOrigin = False
 849
 850     def extractFilesFromCommit(self, commit):
 851         files = []
 852         fnum = 0
 853         while commit.has_key("depotFile%s" % fnum):
 854             path =  commit["depotFile%s" % fnum]
 855
 856             found = [p for p in self.depotPaths
 857                      if path.startswith (p)]
 858             if not found:
 859                 fnum = fnum + 1
 860                 continue
 861
 862             file = {}
 863             file["path"] = path
 864             file["rev"] = commit["rev%s" % fnum]
 865             file["action"] = commit["action%s" % fnum]
 866             file["type"] = commit["type%s" % fnum]
 867             files.append(file)
 868             fnum = fnum + 1
 869         return files
 870
 871     def stripRepoPath(self, path, prefixes):
 872         if self.keepRepoPath:
 873             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 874
 875         for p in prefixes:
 876             if path.startswith(p):
 877                 path = path[len(p):]
 878
 879         return path
 880
 881     def splitFilesIntoBranches(self, commit):
 882         branches = {}
 883         fnum = 0
 884         while commit.has_key("depotFile%s" % fnum):
 885             path =  commit["depotFile%s" % fnum]
 886             found = [p for p in self.depotPaths
 887                      if path.startswith (p)]
 888             if not found:
 889                 fnum = fnum + 1
 890                 continue
 891
 892             file = {}
 893             file["path"] = path
 894             file["rev"] = commit["rev%s" % fnum]
 895             file["action"] = commit["action%s" % fnum]
 896             file["type"] = commit["type%s" % fnum]
 897             fnum = fnum + 1
 898
 899             relPath = self.stripRepoPath(path, self.depotPaths)
 900
 901             for branch in self.knownBranches.keys():
 902
 903                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 904                 if relPath.startswith(branch + "/"):
 905                     if branch not in branches:
 906                         branches[branch] = []
 907                     branches[branch].append(file)
 908                     break
 909
 910         return branches
 911
 912     ## Should move this out, doesn't use SELF.
 913     def readP4Files(self, files):
 914         files = [f for f in files
 915                  if f['action'] != 'delete']
 916
 917         if not files:
 918             return
 919
 920         filedata = p4CmdList('-x - print',
 921                              stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 922                                               for f in files]),
 923                              stdin_mode='w+')
 924         if "p4ExitCode" in filedata[0]:
 925             die("Problems executing p4. Error: [%d]."
 926                 % (filedata[0]['p4ExitCode']));
 927
 928         j = 0;
 929         contents = {}
 930         while j < len(filedata):
 931             stat = filedata[j]
 932             j += 1
 933             text = ''
 934             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 935                 tmp = filedata[j]['data']
 936                 if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 937                     tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 938                 elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 939                     tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 940                 text += tmp
 941                 j += 1
 942
 943
 944             if not stat.has_key('depotFile'):
 945                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 946                 continue
 947
 948             contents[stat['depotFile']] = text
 949
 950         for f in files:
 951             assert not f.has_key('data')
 952             f['data'] = contents[f['path']]
 953
 954     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 955         epoch = details["time"]
 956         author = details["user"]
 957
 958         if self.verbose:
 959             print "commit into %s" % branch
 960
 961         # start with reading files; if that fails, we should not
 962         # create a commit.
 963         new_files = []
 964         for f in files:
 965             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 966                 new_files.append (f)
 967             else:
 968                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 969         files = new_files
 970         self.readP4Files(files)
 971
 972
 973
 974
 975         self.gitStream.write("commit %s\n" % branch)
 976 #        gitStream.write("mark :%s\n" % details["change"])
 977         self.committedChanges.add(int(details["change"]))
 978         committer = ""
 979         if author not in self.users:
 980             self.getUserMapFromPerforceServer()
 981         if author in self.users:
 982             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 983         else:
 984             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 985
 986         self.gitStream.write("committer %s\n" % committer)
 987
 988         self.gitStream.write("data <<EOT\n")
 989         self.gitStream.write(details["desc"])
 990         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 991                              % (','.join (branchPrefixes), details["change"]))
 992         if len(details['options']) > 0:
 993             self.gitStream.write(": options = %s" % details['options'])
 994         self.gitStream.write("]\nEOT\n\n")
 995
 996         if len(parent) > 0:
 997             if self.verbose:
 998                 print "parent %s" % parent
 999             self.gitStream.write("from %s\n" % parent)
1000
1001         for file in files:
1002             if file["type"] == "apple":
1003                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1004                 continue
1005
1006             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1007             if file["action"] == "delete":
1008                 self.gitStream.write("D %s\n" % relPath)
1009             else:
1010                 data = file['data']
1011
1012                 mode = "644"
1013                 if isP4Exec(file["type"]):
1014                     mode = "755"
1015                 elif file["type"] == "symlink":
1016                     mode = "120000"
1017                     # p4 print on a symlink contains "target\n", so strip it off
1018                     data = data[:-1]
1019
1020                 if self.isWindows and file["type"].endswith("text"):
1021                     data = data.replace("\r\n", "\n")
1022
1023                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1024                 self.gitStream.write("data %s\n" % len(data))
1025                 self.gitStream.write(data)
1026                 self.gitStream.write("\n")
1027
1028         self.gitStream.write("\n")
1029
1030         change = int(details["change"])
1031
1032         if self.labels.has_key(change):
1033             label = self.labels[change]
1034             labelDetails = label[0]
1035             labelRevisions = label[1]
1036             if self.verbose:
1037                 print "Change %s is labelled %s" % (change, labelDetails)
1038
1039             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1040                                                     for p in branchPrefixes]))
1041
1042             if len(files) == len(labelRevisions):
1043
1044                 cleanedFiles = {}
1045                 for info in files:
1046                     if info["action"] == "delete":
1047                         continue
1048                     cleanedFiles[info["depotFile"]] = info["rev"]
1049
1050                 if cleanedFiles == labelRevisions:
1051                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1052                     self.gitStream.write("from %s\n" % branch)
1053
1054                     owner = labelDetails["Owner"]
1055                     tagger = ""
1056                     if author in self.users:
1057                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1058                     else:
1059                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1060                     self.gitStream.write("tagger %s\n" % tagger)
1061                     self.gitStream.write("data <<EOT\n")
1062                     self.gitStream.write(labelDetails["Description"])
1063                     self.gitStream.write("EOT\n\n")
1064
1065                 else:
1066                     if not self.silent:
1067                         print ("Tag %s does not match with change %s: files do not match."
1068                                % (labelDetails["label"], change))
1069
1070             else:
1071                 if not self.silent:
1072                     print ("Tag %s does not match with change %s: file count is different."
1073                            % (labelDetails["label"], change))
1074
1075     def getUserCacheFilename(self):
1076         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1077         return home + "/.gitp4-usercache.txt"
1078
1079     def getUserMapFromPerforceServer(self):
1080         if self.userMapFromPerforceServer:
1081             return
1082         self.users = {}
1083
1084         for output in p4CmdList("users"):
1085             if not output.has_key("User"):
1086                 continue
1087             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1088
1089
1090         s = ''
1091         for (key, val) in self.users.items():
1092             s += "%s\t%s\n" % (key, val)
1093
1094         open(self.getUserCacheFilename(), "wb").write(s)
1095         self.userMapFromPerforceServer = True
1096
1097     def loadUserMapFromCache(self):
1098         self.users = {}
1099         self.userMapFromPerforceServer = False
1100         try:
1101             cache = open(self.getUserCacheFilename(), "rb")
1102             lines = cache.readlines()
1103             cache.close()
1104             for line in lines:
1105                 entry = line.strip().split("\t")
1106                 self.users[entry[0]] = entry[1]
1107         except IOError:
1108             self.getUserMapFromPerforceServer()
1109
1110     def getLabels(self):
1111         self.labels = {}
1112
1113         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1114         if len(l) > 0 and not self.silent:
1115             print "Finding files belonging to labels in %s" % `self.depotPaths`
1116
1117         for output in l:
1118             label = output["label"]
1119             revisions = {}
1120             newestChange = 0
1121             if self.verbose:
1122                 print "Querying files for label %s" % label
1123             for file in p4CmdList("files "
1124                                   +  ' '.join (["%s...@%s" % (p, label)
1125                                                 for p in self.depotPaths])):
1126                 revisions[file["depotFile"]] = file["rev"]
1127                 change = int(file["change"])
1128                 if change > newestChange:
1129                     newestChange = change
1130
1131             self.labels[newestChange] = [output, revisions]
1132
1133         if self.verbose:
1134             print "Label changes: %s" % self.labels.keys()
1135
1136     def guessProjectName(self):
1137         for p in self.depotPaths:
1138             if p.endswith("/"):
1139                 p = p[:-1]
1140             p = p[p.strip().rfind("/") + 1:]
1141             if not p.endswith("/"):
1142                p += "/"
1143             return p
1144
1145     def getBranchMapping(self):
1146         lostAndFoundBranches = set()
1147
1148         for info in p4CmdList("branches"):
1149             details = p4Cmd("branch -o %s" % info["branch"])
1150             viewIdx = 0
1151             while details.has_key("View%s" % viewIdx):
1152                 paths = details["View%s" % viewIdx].split(" ")
1153                 viewIdx = viewIdx + 1
1154                 # require standard //depot/foo/... //depot/bar/... mapping
1155                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1156                     continue
1157                 source = paths[0]
1158                 destination = paths[1]
1159                 ## HACK
1160                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1161                     source = source[len(self.depotPaths[0]):-4]
1162                     destination = destination[len(self.depotPaths[0]):-4]
1163
1164                     if destination in self.knownBranches:
1165                         if not self.silent:
1166                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1167                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1168                         continue
1169
1170                     self.knownBranches[destination] = source
1171
1172                     lostAndFoundBranches.discard(destination)
1173
1174                     if source not in self.knownBranches:
1175                         lostAndFoundBranches.add(source)
1176
1177
1178         for branch in lostAndFoundBranches:
1179             self.knownBranches[branch] = branch
1180
1181     def getBranchMappingFromGitBranches(self):
1182         branches = p4BranchesInGit(self.importIntoRemotes)
1183         for branch in branches.keys():
1184             if branch == "master":
1185                 branch = "main"
1186             else:
1187                 branch = branch[len(self.projectName):]
1188             self.knownBranches[branch] = branch
1189
1190     def listExistingP4GitBranches(self):
1191         # branches holds mapping from name to commit
1192         branches = p4BranchesInGit(self.importIntoRemotes)
1193         self.p4BranchesInGit = branches.keys()
1194         for branch in branches.keys():
1195             self.initialParents[self.refPrefix + branch] = branches[branch]
1196
1197     def updateOptionDict(self, d):
1198         option_keys = {}
1199         if self.keepRepoPath:
1200             option_keys['keepRepoPath'] = 1
1201
1202         d["options"] = ' '.join(sorted(option_keys.keys()))
1203
1204     def readOptions(self, d):
1205         self.keepRepoPath = (d.has_key('options')
1206                              and ('keepRepoPath' in d['options']))
1207
1208     def gitRefForBranch(self, branch):
1209         if branch == "main":
1210             return self.refPrefix + "master"
1211
1212         if len(branch) <= 0:
1213             return branch
1214
1215         return self.refPrefix + self.projectName + branch
1216
1217     def gitCommitByP4Change(self, ref, change):
1218         if self.verbose:
1219             print "looking in ref " + ref + " for change %s using bisect..." % change
1220
1221         earliestCommit = ""
1222         latestCommit = parseRevision(ref)
1223
1224         while True:
1225             if self.verbose:
1226                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1227             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1228             if len(next) == 0:
1229                 if self.verbose:
1230                     print "argh"
1231                 return ""
1232             log = extractLogMessageFromGitCommit(next)
1233             settings = extractSettingsGitLog(log)
1234             currentChange = int(settings['change'])
1235             if self.verbose:
1236                 print "current change %s" % currentChange
1237
1238             if currentChange == change:
1239                 if self.verbose:
1240                     print "found %s" % next
1241                 return next
1242
1243             if currentChange < change:
1244                 earliestCommit = "^%s" % next
1245             else:
1246                 latestCommit = "%s" % next
1247
1248         return ""
1249
1250     def importNewBranch(self, branch, maxChange):
1251         # make fast-import flush all changes to disk and update the refs using the checkpoint
1252         # command so that we can try to find the branch parent in the git history
1253         self.gitStream.write("checkpoint\n\n");
1254         self.gitStream.flush();
1255         branchPrefix = self.depotPaths[0] + branch + "/"
1256         range = "@1,%s" % maxChange
1257         #print "prefix" + branchPrefix
1258         changes = p4ChangesForPaths([branchPrefix], range)
1259         if len(changes) <= 0:
1260             return False
1261         firstChange = changes[0]
1262         #print "first change in branch: %s" % firstChange
1263         sourceBranch = self.knownBranches[branch]
1264         sourceDepotPath = self.depotPaths[0] + sourceBranch
1265         sourceRef = self.gitRefForBranch(sourceBranch)
1266         #print "source " + sourceBranch
1267
1268         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1269         #print "branch parent: %s" % branchParentChange
1270         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1271         if len(gitParent) > 0:
1272             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1273             #print "parent git commit: %s" % gitParent
1274
1275         self.importChanges(changes)
1276         return True
1277
1278     def importChanges(self, changes):
1279         cnt = 1
1280         for change in changes:
1281             description = p4Cmd("describe %s" % change)
1282             self.updateOptionDict(description)
1283
1284             if not self.silent:
1285                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1286                 sys.stdout.flush()
1287             cnt = cnt + 1
1288
1289             try:
1290                 if self.detectBranches:
1291                     branches = self.splitFilesIntoBranches(description)
1292                     for branch in branches.keys():
1293                         ## HACK  --hwn
1294                         branchPrefix = self.depotPaths[0] + branch + "/"
1295
1296                         parent = ""
1297
1298                         filesForCommit = branches[branch]
1299
1300                         if self.verbose:
1301                             print "branch is %s" % branch
1302
1303                         self.updatedBranches.add(branch)
1304
1305                         if branch not in self.createdBranches:
1306                             self.createdBranches.add(branch)
1307                             parent = self.knownBranches[branch]
1308                             if parent == branch:
1309                                 parent = ""
1310                             else:
1311                                 fullBranch = self.projectName + branch
1312                                 if fullBranch not in self.p4BranchesInGit:
1313                                     if not self.silent:
1314                                         print("\n    Importing new branch %s" % fullBranch);
1315                                     if self.importNewBranch(branch, change - 1):
1316                                         parent = ""
1317                                         self.p4BranchesInGit.append(fullBranch)
1318                                     if not self.silent:
1319                                         print("\n    Resuming with change %s" % change);
1320
1321                                 if self.verbose:
1322                                     print "parent determined through known branches: %s" % parent
1323
1324                         branch = self.gitRefForBranch(branch)
1325                         parent = self.gitRefForBranch(parent)
1326
1327                         if self.verbose:
1328                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1329
1330                         if len(parent) == 0 and branch in self.initialParents:
1331                             parent = self.initialParents[branch]
1332                             del self.initialParents[branch]
1333
1334                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1335                 else:
1336                     files = self.extractFilesFromCommit(description)
1337                     self.commit(description, files, self.branch, self.depotPaths,
1338                                 self.initialParent)
1339                     self.initialParent = ""
1340             except IOError:
1341                 print self.gitError.read()
1342                 sys.exit(1)
1343
1344     def importHeadRevision(self, revision):
1345         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1346
1347         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1348         details["desc"] = ("Initial import of %s from the state at revision %s"
1349                            % (' '.join(self.depotPaths), revision))
1350         details["change"] = revision
1351         newestRevision = 0
1352
1353         fileCnt = 0
1354         for info in p4CmdList("files "
1355                               +  ' '.join(["%s...%s"
1356                                            % (p, revision)
1357                                            for p in self.depotPaths])):
1358
1359             if info['code'] == 'error':
1360                 sys.stderr.write("p4 returned an error: %s\n"
1361                                  % info['data'])
1362                 sys.exit(1)
1363
1364
1365             change = int(info["change"])
1366             if change > newestRevision:
1367                 newestRevision = change
1368
1369             if info["action"] == "delete":
1370                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1371                 #fileCnt = fileCnt + 1
1372                 continue
1373
1374             for prop in ["depotFile", "rev", "action", "type" ]:
1375                 details["%s%s" % (prop, fileCnt)] = info[prop]
1376
1377             fileCnt = fileCnt + 1
1378
1379         details["change"] = newestRevision
1380         self.updateOptionDict(details)
1381         try:
1382             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1383         except IOError:
1384             print "IO error with git fast-import. Is your git version recent enough?"
1385             print self.gitError.read()
1386
1387
1388     def run(self, args):
1389         self.depotPaths = []
1390         self.changeRange = ""
1391         self.initialParent = ""
1392         self.previousDepotPaths = []
1393
1394         # map from branch depot path to parent branch
1395         self.knownBranches = {}
1396         self.initialParents = {}
1397         self.hasOrigin = originP4BranchesExist()
1398         if not self.syncWithOrigin:
1399             self.hasOrigin = False
1400
1401         if self.importIntoRemotes:
1402             self.refPrefix = "refs/remotes/p4/"
1403         else:
1404             self.refPrefix = "refs/heads/p4/"
1405
1406         if self.syncWithOrigin and self.hasOrigin:
1407             if not self.silent:
1408                 print "Syncing with origin first by calling git fetch origin"
1409             system("git fetch origin")
1410
1411         if len(self.branch) == 0:
1412             self.branch = self.refPrefix + "master"
1413             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1414                 system("git update-ref %s refs/heads/p4" % self.branch)
1415                 system("git branch -D p4");
1416             # create it /after/ importing, when master exists
1417             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1418                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1419
1420         # TODO: should always look at previous commits,
1421         # merge with previous imports, if possible.
1422         if args == []:
1423             if self.hasOrigin:
1424                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1425             self.listExistingP4GitBranches()
1426
1427             if len(self.p4BranchesInGit) > 1:
1428                 if not self.silent:
1429                     print "Importing from/into multiple branches"
1430                 self.detectBranches = True
1431
1432             if self.verbose:
1433                 print "branches: %s" % self.p4BranchesInGit
1434
1435             p4Change = 0
1436             for branch in self.p4BranchesInGit:
1437                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1438
1439                 settings = extractSettingsGitLog(logMsg)
1440
1441                 self.readOptions(settings)
1442                 if (settings.has_key('depot-paths')
1443                     and settings.has_key ('change')):
1444                     change = int(settings['change']) + 1
1445                     p4Change = max(p4Change, change)
1446
1447                     depotPaths = sorted(settings['depot-paths'])
1448                     if self.previousDepotPaths == []:
1449                         self.previousDepotPaths = depotPaths
1450                     else:
1451                         paths = []
1452                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1453                             for i in range(0, min(len(cur), len(prev))):
1454                                 if cur[i] <> prev[i]:
1455                                     i = i - 1
1456                                     break
1457
1458                             paths.append (cur[:i + 1])
1459
1460                         self.previousDepotPaths = paths
1461
1462             if p4Change > 0:
1463                 self.depotPaths = sorted(self.previousDepotPaths)
1464                 self.changeRange = "@%s,#head" % p4Change
1465                 if not self.detectBranches:
1466                     self.initialParent = parseRevision(self.branch)
1467                 if not self.silent and not self.detectBranches:
1468                     print "Performing incremental import into %s git branch" % self.branch
1469
1470         if not self.branch.startswith("refs/"):
1471             self.branch = "refs/heads/" + self.branch
1472
1473         if len(args) == 0 and self.depotPaths:
1474             if not self.silent:
1475                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1476         else:
1477             if self.depotPaths and self.depotPaths != args:
1478                 print ("previous import used depot path %s and now %s was specified. "
1479                        "This doesn't work!" % (' '.join (self.depotPaths),
1480                                                ' '.join (args)))
1481                 sys.exit(1)
1482
1483             self.depotPaths = sorted(args)
1484
1485         revision = ""
1486         self.users = {}
1487
1488         newPaths = []
1489         for p in self.depotPaths:
1490             if p.find("@") != -1:
1491                 atIdx = p.index("@")
1492                 self.changeRange = p[atIdx:]
1493                 if self.changeRange == "@all":
1494                     self.changeRange = ""
1495                 elif ',' not in self.changeRange:
1496                     revision = self.changeRange
1497                     self.changeRange = ""
1498                 p = p[:atIdx]
1499             elif p.find("#") != -1:
1500                 hashIdx = p.index("#")
1501                 revision = p[hashIdx:]
1502                 p = p[:hashIdx]
1503             elif self.previousDepotPaths == []:
1504                 revision = "#head"
1505
1506             p = re.sub ("\.\.\.$", "", p)
1507             if not p.endswith("/"):
1508                 p += "/"
1509
1510             newPaths.append(p)
1511
1512         self.depotPaths = newPaths
1513
1514
1515         self.loadUserMapFromCache()
1516         self.labels = {}
1517         if self.detectLabels:
1518             self.getLabels();
1519
1520         if self.detectBranches:
1521             ## FIXME - what's a P4 projectName ?
1522             self.projectName = self.guessProjectName()
1523
1524             if self.hasOrigin:
1525                 self.getBranchMappingFromGitBranches()
1526             else:
1527                 self.getBranchMapping()
1528             if self.verbose:
1529                 print "p4-git branches: %s" % self.p4BranchesInGit
1530                 print "initial parents: %s" % self.initialParents
1531             for b in self.p4BranchesInGit:
1532                 if b != "master":
1533
1534                     ## FIXME
1535                     b = b[len(self.projectName):]
1536                 self.createdBranches.add(b)
1537
1538         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1539
1540         importProcess = subprocess.Popen(["git", "fast-import"],
1541                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1542                                          stderr=subprocess.PIPE);
1543         self.gitOutput = importProcess.stdout
1544         self.gitStream = importProcess.stdin
1545         self.gitError = importProcess.stderr
1546
1547         if revision:
1548             self.importHeadRevision(revision)
1549         else:
1550             changes = []
1551
1552             if len(self.changesFile) > 0:
1553                 output = open(self.changesFile).readlines()
1554                 changeSet = Set()
1555                 for line in output:
1556                     changeSet.add(int(line))
1557
1558                 for change in changeSet:
1559                     changes.append(change)
1560
1561                 changes.sort()
1562             else:
1563                 if self.verbose:
1564                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1565                                                               self.changeRange)
1566                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1567
1568                 if len(self.maxChanges) > 0:
1569                     changes = changes[:min(int(self.maxChanges), len(changes))]
1570
1571             if len(changes) == 0:
1572                 if not self.silent:
1573                     print "No changes to import!"
1574                 return True
1575
1576             if not self.silent and not self.detectBranches:
1577                 print "Import destination: %s" % self.branch
1578
1579             self.updatedBranches = set()
1580
1581             self.importChanges(changes)
1582
1583             if not self.silent:
1584                 print ""
1585                 if len(self.updatedBranches) > 0:
1586                     sys.stdout.write("Updated branches: ")
1587                     for b in self.updatedBranches:
1588                         sys.stdout.write("%s " % b)
1589                     sys.stdout.write("\n")
1590
1591         self.gitStream.close()
1592         if importProcess.wait() != 0:
1593             die("fast-import failed: %s" % self.gitError.read())
1594         self.gitOutput.close()
1595         self.gitError.close()
1596
1597         return True
1598
1599 class P4Rebase(Command):
1600     def __init__(self):
1601         Command.__init__(self)
1602         self.options = [ ]
1603         self.description = ("Fetches the latest revision from perforce and "
1604                             + "rebases the current work (branch) against it")
1605         self.verbose = False
1606
1607     def run(self, args):
1608         sync = P4Sync()
1609         sync.run([])
1610
1611         return self.rebase()
1612
1613     def rebase(self):
1614         if os.system("git update-index --refresh") != 0:
1615             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1616         if len(read_pipe("git diff-index HEAD --")) > 0:
1617             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1618
1619         [upstream, settings] = findUpstreamBranchPoint()
1620         if len(upstream) == 0:
1621             die("Cannot find upstream branchpoint for rebase")
1622
1623         # the branchpoint may be p4/foo~3, so strip off the parent
1624         upstream = re.sub("~[0-9]+$", "", upstream)
1625
1626         print "Rebasing the current branch onto %s" % upstream
1627         oldHead = read_pipe("git rev-parse HEAD").strip()
1628         system("git rebase %s" % upstream)
1629         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1630         return True
1631
1632 class P4Clone(P4Sync):
1633     def __init__(self):
1634         P4Sync.__init__(self)
1635         self.description = "Creates a new git repository and imports from Perforce into it"
1636         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1637         self.options.append(
1638             optparse.make_option("--destination", dest="cloneDestination",
1639                                  action='store', default=None,
1640                                  help="where to leave result of the clone"))
1641         self.cloneDestination = None
1642         self.needsGit = False
1643
1644     def defaultDestination(self, args):
1645         ## TODO: use common prefix of args?
1646         depotPath = args[0]
1647         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1648         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1649         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1650         depotDir = re.sub(r"/$", "", depotDir)
1651         return os.path.split(depotDir)[1]
1652
1653     def run(self, args):
1654         if len(args) < 1:
1655             return False
1656
1657         if self.keepRepoPath and not self.cloneDestination:
1658             sys.stderr.write("Must specify destination for --keep-path\n")
1659             sys.exit(1)
1660
1661         depotPaths = args
1662
1663         if not self.cloneDestination and len(depotPaths) > 1:
1664             self.cloneDestination = depotPaths[-1]
1665             depotPaths = depotPaths[:-1]
1666
1667         for p in depotPaths:
1668             if not p.startswith("//"):
1669                 return False
1670
1671         if not self.cloneDestination:
1672             self.cloneDestination = self.defaultDestination(args)
1673
1674         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1675         if not os.path.exists(self.cloneDestination):
1676             os.makedirs(self.cloneDestination)
1677         os.chdir(self.cloneDestination)
1678         system("git init")
1679         self.gitdir = os.getcwd() + "/.git"
1680         if not P4Sync.run(self, depotPaths):
1681             return False
1682         if self.branch != "master":
1683             if gitBranchExists("refs/remotes/p4/master"):
1684                 system("git branch master refs/remotes/p4/master")
1685                 system("git checkout -f")
1686             else:
1687                 print "Could not detect main branch. No checkout/master branch created."
1688
1689         return True
1690
1691 class P4Branches(Command):
1692     def __init__(self):
1693         Command.__init__(self)
1694         self.options = [ ]
1695         self.description = ("Shows the git branches that hold imports and their "
1696                             + "corresponding perforce depot paths")
1697         self.verbose = False
1698
1699     def run(self, args):
1700         if originP4BranchesExist():
1701             createOrUpdateBranchesFromOrigin()
1702
1703         cmdline = "git rev-parse --symbolic "
1704         cmdline += " --remotes"
1705
1706         for line in read_pipe_lines(cmdline):
1707             line = line.strip()
1708
1709             if not line.startswith('p4/') or line == "p4/HEAD":
1710                 continue
1711             branch = line
1712
1713             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1714             settings = extractSettingsGitLog(log)
1715
1716             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1717         return True
1718
1719 class HelpFormatter(optparse.IndentedHelpFormatter):
1720     def __init__(self):
1721         optparse.IndentedHelpFormatter.__init__(self)
1722
1723     def format_description(self, description):
1724         if description:
1725             return description + "\n"
1726         else:
1727             return ""
1728
1729 def printUsage(commands):
1730     print "usage: %s <command> [options]" % sys.argv[0]
1731     print ""
1732     print "valid commands: %s" % ", ".join(commands)
1733     print ""
1734     print "Try %s <command> --help for command specific help." % sys.argv[0]
1735     print ""
1736
1737 commands = {
1738     "debug" : P4Debug,
1739     "submit" : P4Submit,
1740     "commit" : P4Submit,
1741     "sync" : P4Sync,
1742     "rebase" : P4Rebase,
1743     "clone" : P4Clone,
1744     "rollback" : P4RollBack,
1745     "branches" : P4Branches
1746 }
1747
1748
1749 def main():
1750     if len(sys.argv[1:]) == 0:
1751         printUsage(commands.keys())
1752         sys.exit(2)
1753
1754     cmd = ""
1755     cmdName = sys.argv[1]
1756     try:
1757         klass = commands[cmdName]
1758         cmd = klass()
1759     except KeyError:
1760         print "unknown command %s" % cmdName
1761         print ""
1762         printUsage(commands.keys())
1763         sys.exit(2)
1764
1765     options = cmd.options
1766     cmd.gitdir = os.environ.get("GIT_DIR", None)
1767
1768     args = sys.argv[2:]
1769
1770     if len(options) > 0:
1771         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1772
1773         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1774                                        options,
1775                                        description = cmd.description,
1776                                        formatter = HelpFormatter())
1777
1778         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1779     global verbose
1780     verbose = cmd.verbose
1781     if cmd.needsGit:
1782         if cmd.gitdir == None:
1783             cmd.gitdir = os.path.abspath(".git")
1784             if not isValidGitDir(cmd.gitdir):
1785                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1786                 if os.path.exists(cmd.gitdir):
1787                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1788                     if len(cdup) > 0:
1789                         os.chdir(cdup);
1790
1791         if not isValidGitDir(cmd.gitdir):
1792             if isValidGitDir(cmd.gitdir + "/.git"):
1793                 cmd.gitdir += "/.git"
1794             else:
1795                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1796
1797         os.environ["GIT_DIR"] = cmd.gitdir
1798
1799     if not cmd.run(args):
1800         parser.print_help()
1801
1802
1803 if __name__ == '__main__':
1804     main()