contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s %s" % ("p4", cmd)
  28     if verbose:
  29         print real_cmd
  30     return real_cmd
  31
  32 def die(msg):
  33     if verbose:
  34         raise Exception(msg)
  35     else:
  36         sys.stderr.write(msg + "\n")
  37         sys.exit(1)
  38
  39 def write_pipe(c, str):
  40     if verbose:
  41         sys.stderr.write('Writing pipe: %s\n' % c)
  42
  43     pipe = os.popen(c, 'w')
  44     val = pipe.write(str)
  45     if pipe.close():
  46         die('Command failed: %s' % c)
  47
  48     return val
  49
  50 def read_pipe(c, ignore_error=False):
  51     if verbose:
  52         sys.stderr.write('Reading pipe: %s\n' % c)
  53
  54     pipe = os.popen(c, 'rb')
  55     val = pipe.read()
  56     if pipe.close() and not ignore_error:
  57         die('Command failed: %s' % c)
  58
  59     return val
  60
  61
  62 def read_pipe_lines(c):
  63     if verbose:
  64         sys.stderr.write('Reading pipe: %s\n' % c)
  65     ## todo: check return status
  66     pipe = os.popen(c, 'rb')
  67     val = pipe.readlines()
  68     if pipe.close():
  69         die('Command failed: %s' % c)
  70
  71     return val
  72
  73 def p4_read_pipe_lines(c):
  74     """Specifically invoke p4 on the command supplied. """
  75     real_cmd = "%s %s" % ("p4", c)
  76     if verbose:
  77         print real_cmd
  78     return read_pipe_lines(real_cmd)
  79
  80 def system(cmd):
  81     if verbose:
  82         sys.stderr.write("executing %s\n" % cmd)
  83     if os.system(cmd) != 0:
  84         die("command failed: %s" % cmd)
  85
  86 def p4_system(cmd):
  87     """Specifically invoke p4 as the system command. """
  88     real_cmd = "%s %s" % ("p4", cmd)
  89     if verbose:
  90         print real_cmd
  91     return system(real_cmd)
  92
  93 def isP4Exec(kind):
  94     """Determine if a Perforce 'kind' should have execute permission
  95
  96     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  97     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  98     a plus sign, it is also executable"""
  99     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 100
 101 def setP4ExecBit(file, mode):
 102     # Reopens an already open file and changes the execute bit to match
 103     # the execute bit setting in the passed in mode.
 104
 105     p4Type = "+x"
 106
 107     if not isModeExec(mode):
 108         p4Type = getP4OpenedType(file)
 109         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 110         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 111         if p4Type[-1] == "+":
 112             p4Type = p4Type[0:-1]
 113
 114     p4_system("reopen -t %s %s" % (p4Type, file))
 115
 116 def getP4OpenedType(file):
 117     # Returns the perforce file type for the given file.
 118
 119     result = read_pipe("p4 opened %s" % file)
 120     match = re.match(".*\((.+)\)\r?$", result)
 121     if match:
 122         return match.group(1)
 123     else:
 124         die("Could not determine file type for %s (result: '%s')" % (file, result))
 125
 126 def diffTreePattern():
 127     # This is a simple generator for the diff tree regex pattern. This could be
 128     # a class variable if this and parseDiffTreeEntry were a part of a class.
 129     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 130     while True:
 131         yield pattern
 132
 133 def parseDiffTreeEntry(entry):
 134     """Parses a single diff tree entry into its component elements.
 135
 136     See git-diff-tree(1) manpage for details about the format of the diff
 137     output. This method returns a dictionary with the following elements:
 138
 139     src_mode - The mode of the source file
 140     dst_mode - The mode of the destination file
 141     src_sha1 - The sha1 for the source file
 142     dst_sha1 - The sha1 fr the destination file
 143     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 144     status_score - The score for the status (applicable for 'C' and 'R'
 145                    statuses). This is None if there is no score.
 146     src - The path for the source file.
 147     dst - The path for the destination file. This is only present for
 148           copy or renames. If it is not present, this is None.
 149
 150     If the pattern is not matched, None is returned."""
 151
 152     match = diffTreePattern().next().match(entry)
 153     if match:
 154         return {
 155             'src_mode': match.group(1),
 156             'dst_mode': match.group(2),
 157             'src_sha1': match.group(3),
 158             'dst_sha1': match.group(4),
 159             'status': match.group(5),
 160             'status_score': match.group(6),
 161             'src': match.group(7),
 162             'dst': match.group(10)
 163         }
 164     return None
 165
 166 def isModeExec(mode):
 167     # Returns True if the given git mode represents an executable file,
 168     # otherwise False.
 169     return mode[-3:] == "755"
 170
 171 def isModeExecChanged(src_mode, dst_mode):
 172     return isModeExec(src_mode) != isModeExec(dst_mode)
 173
 174 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 175     cmd = "p4 -G %s" % cmd
 176     if verbose:
 177         sys.stderr.write("Opening pipe: %s\n" % cmd)
 178
 179     # Use a temporary file to avoid deadlocks without
 180     # subprocess.communicate(), which would put another copy
 181     # of stdout into memory.
 182     stdin_file = None
 183     if stdin is not None:
 184         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 185         stdin_file.write(stdin)
 186         stdin_file.flush()
 187         stdin_file.seek(0)
 188
 189     p4 = subprocess.Popen(cmd, shell=True,
 190                           stdin=stdin_file,
 191                           stdout=subprocess.PIPE)
 192
 193     result = []
 194     try:
 195         while True:
 196             entry = marshal.load(p4.stdout)
 197             result.append(entry)
 198     except EOFError:
 199         pass
 200     exitCode = p4.wait()
 201     if exitCode != 0:
 202         entry = {}
 203         entry["p4ExitCode"] = exitCode
 204         result.append(entry)
 205
 206     return result
 207
 208 def p4Cmd(cmd):
 209     list = p4CmdList(cmd)
 210     result = {}
 211     for entry in list:
 212         result.update(entry)
 213     return result;
 214
 215 def p4Where(depotPath):
 216     if not depotPath.endswith("/"):
 217         depotPath += "/"
 218     output = p4Cmd("where %s..." % depotPath)
 219     if output["code"] == "error":
 220         return ""
 221     clientPath = ""
 222     if "path" in output:
 223         clientPath = output.get("path")
 224     elif "data" in output:
 225         data = output.get("data")
 226         lastSpace = data.rfind(" ")
 227         clientPath = data[lastSpace + 1:]
 228
 229     if clientPath.endswith("..."):
 230         clientPath = clientPath[:-3]
 231     return clientPath
 232
 233 def currentGitBranch():
 234     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 235
 236 def isValidGitDir(path):
 237     if (os.path.exists(path + "/HEAD")
 238         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 239         return True;
 240     return False
 241
 242 def parseRevision(ref):
 243     return read_pipe("git rev-parse %s" % ref).strip()
 244
 245 def extractLogMessageFromGitCommit(commit):
 246     logMessage = ""
 247
 248     ## fixme: title is first line of commit, not 1st paragraph.
 249     foundTitle = False
 250     for log in read_pipe_lines("git cat-file commit %s" % commit):
 251        if not foundTitle:
 252            if len(log) == 1:
 253                foundTitle = True
 254            continue
 255
 256        logMessage += log
 257     return logMessage
 258
 259 def extractSettingsGitLog(log):
 260     values = {}
 261     for line in log.split("\n"):
 262         line = line.strip()
 263         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 264         if not m:
 265             continue
 266
 267         assignments = m.group(1).split (':')
 268         for a in assignments:
 269             vals = a.split ('=')
 270             key = vals[0].strip()
 271             val = ('='.join (vals[1:])).strip()
 272             if val.endswith ('\"') and val.startswith('"'):
 273                 val = val[1:-1]
 274
 275             values[key] = val
 276
 277     paths = values.get("depot-paths")
 278     if not paths:
 279         paths = values.get("depot-path")
 280     if paths:
 281         values['depot-paths'] = paths.split(',')
 282     return values
 283
 284 def gitBranchExists(branch):
 285     proc = subprocess.Popen(["git", "rev-parse", branch],
 286                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 287     return proc.wait() == 0;
 288
 289 def gitConfig(key):
 290     return read_pipe("git config %s" % key, ignore_error=True).strip()
 291
 292 def p4BranchesInGit(branchesAreInRemotes = True):
 293     branches = {}
 294
 295     cmdline = "git rev-parse --symbolic "
 296     if branchesAreInRemotes:
 297         cmdline += " --remotes"
 298     else:
 299         cmdline += " --branches"
 300
 301     for line in read_pipe_lines(cmdline):
 302         line = line.strip()
 303
 304         ## only import to p4/
 305         if not line.startswith('p4/') or line == "p4/HEAD":
 306             continue
 307         branch = line
 308
 309         # strip off p4
 310         branch = re.sub ("^p4/", "", line)
 311
 312         branches[branch] = parseRevision(line)
 313     return branches
 314
 315 def findUpstreamBranchPoint(head = "HEAD"):
 316     branches = p4BranchesInGit()
 317     # map from depot-path to branch name
 318     branchByDepotPath = {}
 319     for branch in branches.keys():
 320         tip = branches[branch]
 321         log = extractLogMessageFromGitCommit(tip)
 322         settings = extractSettingsGitLog(log)
 323         if settings.has_key("depot-paths"):
 324             paths = ",".join(settings["depot-paths"])
 325             branchByDepotPath[paths] = "remotes/p4/" + branch
 326
 327     settings = None
 328     parent = 0
 329     while parent < 65535:
 330         commit = head + "~%s" % parent
 331         log = extractLogMessageFromGitCommit(commit)
 332         settings = extractSettingsGitLog(log)
 333         if settings.has_key("depot-paths"):
 334             paths = ",".join(settings["depot-paths"])
 335             if branchByDepotPath.has_key(paths):
 336                 return [branchByDepotPath[paths], settings]
 337
 338         parent = parent + 1
 339
 340     return ["", settings]
 341
 342 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 343     if not silent:
 344         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 345                % localRefPrefix)
 346
 347     originPrefix = "origin/p4/"
 348
 349     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 350         line = line.strip()
 351         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 352             continue
 353
 354         headName = line[len(originPrefix):]
 355         remoteHead = localRefPrefix + headName
 356         originHead = line
 357
 358         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 359         if (not original.has_key('depot-paths')
 360             or not original.has_key('change')):
 361             continue
 362
 363         update = False
 364         if not gitBranchExists(remoteHead):
 365             if verbose:
 366                 print "creating %s" % remoteHead
 367             update = True
 368         else:
 369             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 370             if settings.has_key('change') > 0:
 371                 if settings['depot-paths'] == original['depot-paths']:
 372                     originP4Change = int(original['change'])
 373                     p4Change = int(settings['change'])
 374                     if originP4Change > p4Change:
 375                         print ("%s (%s) is newer than %s (%s). "
 376                                "Updating p4 branch from origin."
 377                                % (originHead, originP4Change,
 378                                   remoteHead, p4Change))
 379                         update = True
 380                 else:
 381                     print ("Ignoring: %s was imported from %s while "
 382                            "%s was imported from %s"
 383                            % (originHead, ','.join(original['depot-paths']),
 384                               remoteHead, ','.join(settings['depot-paths'])))
 385
 386         if update:
 387             system("git update-ref %s %s" % (remoteHead, originHead))
 388
 389 def originP4BranchesExist():
 390         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 391
 392 def p4ChangesForPaths(depotPaths, changeRange):
 393     assert depotPaths
 394     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 395                                                         for p in depotPaths]))
 396
 397     changes = []
 398     for line in output:
 399         changeNum = line.split(" ")[1]
 400         changes.append(int(changeNum))
 401
 402     changes.sort()
 403     return changes
 404
 405 class Command:
 406     def __init__(self):
 407         self.usage = "usage: %prog [options]"
 408         self.needsGit = True
 409
 410 class P4Debug(Command):
 411     def __init__(self):
 412         Command.__init__(self)
 413         self.options = [
 414             optparse.make_option("--verbose", dest="verbose", action="store_true",
 415                                  default=False),
 416             ]
 417         self.description = "A tool to debug the output of p4 -G."
 418         self.needsGit = False
 419         self.verbose = False
 420
 421     def run(self, args):
 422         j = 0
 423         for output in p4CmdList(" ".join(args)):
 424             print 'Element: %d' % j
 425             j += 1
 426             print output
 427         return True
 428
 429 class P4RollBack(Command):
 430     def __init__(self):
 431         Command.__init__(self)
 432         self.options = [
 433             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 434             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 435         ]
 436         self.description = "A tool to debug the multi-branch import. Don't use :)"
 437         self.verbose = False
 438         self.rollbackLocalBranches = False
 439
 440     def run(self, args):
 441         if len(args) != 1:
 442             return False
 443         maxChange = int(args[0])
 444
 445         if "p4ExitCode" in p4Cmd("changes -m 1"):
 446             die("Problems executing p4");
 447
 448         if self.rollbackLocalBranches:
 449             refPrefix = "refs/heads/"
 450             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 451         else:
 452             refPrefix = "refs/remotes/"
 453             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 454
 455         for line in lines:
 456             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 457                 line = line.strip()
 458                 ref = refPrefix + line
 459                 log = extractLogMessageFromGitCommit(ref)
 460                 settings = extractSettingsGitLog(log)
 461
 462                 depotPaths = settings['depot-paths']
 463                 change = settings['change']
 464
 465                 changed = False
 466
 467                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 468                                                            for p in depotPaths]))) == 0:
 469                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 470                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 471                     continue
 472
 473                 while change and int(change) > maxChange:
 474                     changed = True
 475                     if self.verbose:
 476                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 477                     system("git update-ref %s \"%s^\"" % (ref, ref))
 478                     log = extractLogMessageFromGitCommit(ref)
 479                     settings =  extractSettingsGitLog(log)
 480
 481
 482                     depotPaths = settings['depot-paths']
 483                     change = settings['change']
 484
 485                 if changed:
 486                     print "%s rewound to %s" % (ref, change)
 487
 488         return True
 489
 490 class P4Submit(Command):
 491     def __init__(self):
 492         Command.__init__(self)
 493         self.options = [
 494                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 495                 optparse.make_option("--origin", dest="origin"),
 496                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 497         ]
 498         self.description = "Submit changes from git to the perforce depot."
 499         self.usage += " [name of git branch to submit into perforce depot]"
 500         self.interactive = True
 501         self.origin = ""
 502         self.detectRename = False
 503         self.verbose = False
 504         self.isWindows = (platform.system() == "Windows")
 505
 506     def check(self):
 507         if len(p4CmdList("opened ...")) > 0:
 508             die("You have files opened with perforce! Close them before starting the sync.")
 509
 510     # replaces everything between 'Description:' and the next P4 submit template field with the
 511     # commit message
 512     def prepareLogMessage(self, template, message):
 513         result = ""
 514
 515         inDescriptionSection = False
 516
 517         for line in template.split("\n"):
 518             if line.startswith("#"):
 519                 result += line + "\n"
 520                 continue
 521
 522             if inDescriptionSection:
 523                 if line.startswith("Files:"):
 524                     inDescriptionSection = False
 525                 else:
 526                     continue
 527             else:
 528                 if line.startswith("Description:"):
 529                     inDescriptionSection = True
 530                     line += "\n"
 531                     for messageLine in message.split("\n"):
 532                         line += "\t" + messageLine + "\n"
 533
 534             result += line + "\n"
 535
 536         return result
 537
 538     def prepareSubmitTemplate(self):
 539         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 540         template = ""
 541         inFilesSection = False
 542         for line in p4_read_pipe_lines("change -o"):
 543             if line.endswith("\r\n"):
 544                 line = line[:-2] + "\n"
 545             if inFilesSection:
 546                 if line.startswith("\t"):
 547                     # path starts and ends with a tab
 548                     path = line[1:]
 549                     lastTab = path.rfind("\t")
 550                     if lastTab != -1:
 551                         path = path[:lastTab]
 552                         if not path.startswith(self.depotPath):
 553                             continue
 554                 else:
 555                     inFilesSection = False
 556             else:
 557                 if line.startswith("Files:"):
 558                     inFilesSection = True
 559
 560             template += line
 561
 562         return template
 563
 564     def applyCommit(self, id):
 565         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 566         diffOpts = ("", "-M")[self.detectRename]
 567         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 568         filesToAdd = set()
 569         filesToDelete = set()
 570         editedFiles = set()
 571         filesToChangeExecBit = {}
 572         for line in diff:
 573             diff = parseDiffTreeEntry(line)
 574             modifier = diff['status']
 575             path = diff['src']
 576             if modifier == "M":
 577                 p4_system("edit \"%s\"" % path)
 578                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 579                     filesToChangeExecBit[path] = diff['dst_mode']
 580                 editedFiles.add(path)
 581             elif modifier == "A":
 582                 filesToAdd.add(path)
 583                 filesToChangeExecBit[path] = diff['dst_mode']
 584                 if path in filesToDelete:
 585                     filesToDelete.remove(path)
 586             elif modifier == "D":
 587                 filesToDelete.add(path)
 588                 if path in filesToAdd:
 589                     filesToAdd.remove(path)
 590             elif modifier == "R":
 591                 src, dest = diff['src'], diff['dst']
 592                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 593                 p4_system("edit \"%s\"" % (dest))
 594                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 595                     filesToChangeExecBit[dest] = diff['dst_mode']
 596                 os.unlink(dest)
 597                 editedFiles.add(dest)
 598                 filesToDelete.add(src)
 599             else:
 600                 die("unknown modifier %s for %s" % (modifier, path))
 601
 602         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 603         patchcmd = diffcmd + " | git apply "
 604         tryPatchCmd = patchcmd + "--check -"
 605         applyPatchCmd = patchcmd + "--check --apply -"
 606
 607         if os.system(tryPatchCmd) != 0:
 608             print "Unfortunately applying the change failed!"
 609             print "What do you want to do?"
 610             response = "x"
 611             while response != "s" and response != "a" and response != "w":
 612                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 613                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 614             if response == "s":
 615                 print "Skipping! Good luck with the next patches..."
 616                 for f in editedFiles:
 617                     p4_system("revert \"%s\"" % f);
 618                 for f in filesToAdd:
 619                     system("rm %s" %f)
 620                 return
 621             elif response == "a":
 622                 os.system(applyPatchCmd)
 623                 if len(filesToAdd) > 0:
 624                     print "You may also want to call p4 add on the following files:"
 625                     print " ".join(filesToAdd)
 626                 if len(filesToDelete):
 627                     print "The following files should be scheduled for deletion with p4 delete:"
 628                     print " ".join(filesToDelete)
 629                 die("Please resolve and submit the conflict manually and "
 630                     + "continue afterwards with git-p4 submit --continue")
 631             elif response == "w":
 632                 system(diffcmd + " > patch.txt")
 633                 print "Patch saved to patch.txt in %s !" % self.clientPath
 634                 die("Please resolve and submit the conflict manually and "
 635                     "continue afterwards with git-p4 submit --continue")
 636
 637         system(applyPatchCmd)
 638
 639         for f in filesToAdd:
 640             p4_system("add \"%s\"" % f)
 641         for f in filesToDelete:
 642             p4_system("revert \"%s\"" % f)
 643             p4_system("delete \"%s\"" % f)
 644
 645         # Set/clear executable bits
 646         for f in filesToChangeExecBit.keys():
 647             mode = filesToChangeExecBit[f]
 648             setP4ExecBit(f, mode)
 649
 650         logMessage = extractLogMessageFromGitCommit(id)
 651         logMessage = logMessage.strip()
 652
 653         template = self.prepareSubmitTemplate()
 654
 655         if self.interactive:
 656             submitTemplate = self.prepareLogMessage(template, logMessage)
 657             if os.environ.has_key("P4DIFF"):
 658                 del(os.environ["P4DIFF"])
 659             diff = read_pipe("p4 diff -du ...")
 660
 661             newdiff = ""
 662             for newFile in filesToAdd:
 663                 newdiff += "==== new file ====\n"
 664                 newdiff += "--- /dev/null\n"
 665                 newdiff += "+++ %s\n" % newFile
 666                 f = open(newFile, "r")
 667                 for line in f.readlines():
 668                     newdiff += "+" + line
 669                 f.close()
 670
 671             separatorLine = "######## everything below this line is just the diff #######\n"
 672
 673             [handle, fileName] = tempfile.mkstemp()
 674             tmpFile = os.fdopen(handle, "w+")
 675             if self.isWindows:
 676                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 677                 separatorLine = separatorLine.replace("\n", "\r\n")
 678                 newdiff = newdiff.replace("\n", "\r\n")
 679             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 680             tmpFile.close()
 681             defaultEditor = "vi"
 682             if platform.system() == "Windows":
 683                 defaultEditor = "notepad"
 684             if os.environ.has_key("P4EDITOR"):
 685                 editor = os.environ.get("P4EDITOR")
 686             else:
 687                 editor = os.environ.get("EDITOR", defaultEditor);
 688             system(editor + " " + fileName)
 689             tmpFile = open(fileName, "rb")
 690             message = tmpFile.read()
 691             tmpFile.close()
 692             os.remove(fileName)
 693             submitTemplate = message[:message.index(separatorLine)]
 694             if self.isWindows:
 695                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 696
 697             write_pipe("p4 submit -i", submitTemplate)
 698         else:
 699             fileName = "submit.txt"
 700             file = open(fileName, "w+")
 701             file.write(self.prepareLogMessage(template, logMessage))
 702             file.close()
 703             print ("Perforce submit template written as %s. "
 704                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 705                    % (fileName, fileName))
 706
 707     def run(self, args):
 708         if len(args) == 0:
 709             self.master = currentGitBranch()
 710             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 711                 die("Detecting current git branch failed!")
 712         elif len(args) == 1:
 713             self.master = args[0]
 714         else:
 715             return False
 716
 717         allowSubmit = gitConfig("git-p4.allowSubmit")
 718         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 719             die("%s is not in git-p4.allowSubmit" % self.master)
 720
 721         [upstream, settings] = findUpstreamBranchPoint()
 722         self.depotPath = settings['depot-paths'][0]
 723         if len(self.origin) == 0:
 724             self.origin = upstream
 725
 726         if self.verbose:
 727             print "Origin branch is " + self.origin
 728
 729         if len(self.depotPath) == 0:
 730             print "Internal error: cannot locate perforce depot path from existing branches"
 731             sys.exit(128)
 732
 733         self.clientPath = p4Where(self.depotPath)
 734
 735         if len(self.clientPath) == 0:
 736             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 737             sys.exit(128)
 738
 739         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 740         self.oldWorkingDirectory = os.getcwd()
 741
 742         os.chdir(self.clientPath)
 743         print "Syncronizing p4 checkout..."
 744         p4_system("sync ...")
 745
 746         self.check()
 747
 748         commits = []
 749         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 750             commits.append(line.strip())
 751         commits.reverse()
 752
 753         while len(commits) > 0:
 754             commit = commits[0]
 755             commits = commits[1:]
 756             self.applyCommit(commit)
 757             if not self.interactive:
 758                 break
 759
 760         if len(commits) == 0:
 761             print "All changes applied!"
 762             os.chdir(self.oldWorkingDirectory)
 763
 764             sync = P4Sync()
 765             sync.run([])
 766
 767             rebase = P4Rebase()
 768             rebase.rebase()
 769
 770         return True
 771
 772 class P4Sync(Command):
 773     def __init__(self):
 774         Command.__init__(self)
 775         self.options = [
 776                 optparse.make_option("--branch", dest="branch"),
 777                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 778                 optparse.make_option("--changesfile", dest="changesFile"),
 779                 optparse.make_option("--silent", dest="silent", action="store_true"),
 780                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 781                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 782                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 783                                      help="Import into refs/heads/ , not refs/remotes"),
 784                 optparse.make_option("--max-changes", dest="maxChanges"),
 785                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 786                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 787                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 788                                      help="Only sync files that are included in the Perforce Client Spec")
 789         ]
 790         self.description = """Imports from Perforce into a git repository.\n
 791     example:
 792     //depot/my/project/ -- to import the current head
 793     //depot/my/project/@all -- to import everything
 794     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 795
 796     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 797
 798         self.usage += " //depot/path[@revRange]"
 799         self.silent = False
 800         self.createdBranches = Set()
 801         self.committedChanges = Set()
 802         self.branch = ""
 803         self.detectBranches = False
 804         self.detectLabels = False
 805         self.changesFile = ""
 806         self.syncWithOrigin = True
 807         self.verbose = False
 808         self.importIntoRemotes = True
 809         self.maxChanges = ""
 810         self.isWindows = (platform.system() == "Windows")
 811         self.keepRepoPath = False
 812         self.depotPaths = None
 813         self.p4BranchesInGit = []
 814         self.cloneExclude = []
 815         self.useClientSpec = False
 816         self.clientSpecDirs = []
 817
 818         if gitConfig("git-p4.syncFromOrigin") == "false":
 819             self.syncWithOrigin = False
 820
 821     def extractFilesFromCommit(self, commit):
 822         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 823                              for path in self.cloneExclude]
 824         files = []
 825         fnum = 0
 826         while commit.has_key("depotFile%s" % fnum):
 827             path =  commit["depotFile%s" % fnum]
 828
 829             if [p for p in self.cloneExclude
 830                 if path.startswith (p)]:
 831                 found = False
 832             else:
 833                 found = [p for p in self.depotPaths
 834                          if path.startswith (p)]
 835             if not found:
 836                 fnum = fnum + 1
 837                 continue
 838
 839             file = {}
 840             file["path"] = path
 841             file["rev"] = commit["rev%s" % fnum]
 842             file["action"] = commit["action%s" % fnum]
 843             file["type"] = commit["type%s" % fnum]
 844             files.append(file)
 845             fnum = fnum + 1
 846         return files
 847
 848     def stripRepoPath(self, path, prefixes):
 849         if self.keepRepoPath:
 850             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 851
 852         for p in prefixes:
 853             if path.startswith(p):
 854                 path = path[len(p):]
 855
 856         return path
 857
 858     def splitFilesIntoBranches(self, commit):
 859         branches = {}
 860         fnum = 0
 861         while commit.has_key("depotFile%s" % fnum):
 862             path =  commit["depotFile%s" % fnum]
 863             found = [p for p in self.depotPaths
 864                      if path.startswith (p)]
 865             if not found:
 866                 fnum = fnum + 1
 867                 continue
 868
 869             file = {}
 870             file["path"] = path
 871             file["rev"] = commit["rev%s" % fnum]
 872             file["action"] = commit["action%s" % fnum]
 873             file["type"] = commit["type%s" % fnum]
 874             fnum = fnum + 1
 875
 876             relPath = self.stripRepoPath(path, self.depotPaths)
 877
 878             for branch in self.knownBranches.keys():
 879
 880                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 881                 if relPath.startswith(branch + "/"):
 882                     if branch not in branches:
 883                         branches[branch] = []
 884                     branches[branch].append(file)
 885                     break
 886
 887         return branches
 888
 889     ## Should move this out, doesn't use SELF.
 890     def readP4Files(self, files):
 891         filesForCommit = []
 892         filesToRead = []
 893
 894         for f in files:
 895             includeFile = True
 896             for val in self.clientSpecDirs:
 897                 if f['path'].startswith(val[0]):
 898                     if val[1] <= 0:
 899                         includeFile = False
 900                     break
 901
 902             if includeFile:
 903                 filesForCommit.append(f)
 904                 if f['action'] != 'delete':
 905                     filesToRead.append(f)
 906
 907         filedata = []
 908         if len(filesToRead) > 0:
 909             filedata = p4CmdList('-x - print',
 910                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 911                                                   for f in filesToRead]),
 912                                  stdin_mode='w+')
 913
 914             if "p4ExitCode" in filedata[0]:
 915                 die("Problems executing p4. Error: [%d]."
 916                     % (filedata[0]['p4ExitCode']));
 917
 918         j = 0;
 919         contents = {}
 920         while j < len(filedata):
 921             stat = filedata[j]
 922             j += 1
 923             text = [];
 924             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 925                 text.append(filedata[j]['data'])
 926                 j += 1
 927             text = ''.join(text)
 928
 929             if not stat.has_key('depotFile'):
 930                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 931                 continue
 932
 933             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 934                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 935             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 936                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 937
 938             contents[stat['depotFile']] = text
 939
 940         for f in filesForCommit:
 941             path = f['path']
 942             if contents.has_key(path):
 943                 f['data'] = contents[path]
 944
 945         return filesForCommit
 946
 947     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 948         epoch = details["time"]
 949         author = details["user"]
 950
 951         if self.verbose:
 952             print "commit into %s" % branch
 953
 954         # start with reading files; if that fails, we should not
 955         # create a commit.
 956         new_files = []
 957         for f in files:
 958             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 959                 new_files.append (f)
 960             else:
 961                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 962         files = self.readP4Files(new_files)
 963
 964         self.gitStream.write("commit %s\n" % branch)
 965 #        gitStream.write("mark :%s\n" % details["change"])
 966         self.committedChanges.add(int(details["change"]))
 967         committer = ""
 968         if author not in self.users:
 969             self.getUserMapFromPerforceServer()
 970         if author in self.users:
 971             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 972         else:
 973             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 974
 975         self.gitStream.write("committer %s\n" % committer)
 976
 977         self.gitStream.write("data <<EOT\n")
 978         self.gitStream.write(details["desc"])
 979         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 980                              % (','.join (branchPrefixes), details["change"]))
 981         if len(details['options']) > 0:
 982             self.gitStream.write(": options = %s" % details['options'])
 983         self.gitStream.write("]\nEOT\n\n")
 984
 985         if len(parent) > 0:
 986             if self.verbose:
 987                 print "parent %s" % parent
 988             self.gitStream.write("from %s\n" % parent)
 989
 990         for file in files:
 991             if file["type"] == "apple":
 992                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 993                 continue
 994
 995             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 996             if file["action"] == "delete":
 997                 self.gitStream.write("D %s\n" % relPath)
 998             else:
 999                 data = file['data']
1000
1001                 mode = "644"
1002                 if isP4Exec(file["type"]):
1003                     mode = "755"
1004                 elif file["type"] == "symlink":
1005                     mode = "120000"
1006                     # p4 print on a symlink contains "target\n", so strip it off
1007                     data = data[:-1]
1008
1009                 if self.isWindows and file["type"].endswith("text"):
1010                     data = data.replace("\r\n", "\n")
1011
1012                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1013                 self.gitStream.write("data %s\n" % len(data))
1014                 self.gitStream.write(data)
1015                 self.gitStream.write("\n")
1016
1017         self.gitStream.write("\n")
1018
1019         change = int(details["change"])
1020
1021         if self.labels.has_key(change):
1022             label = self.labels[change]
1023             labelDetails = label[0]
1024             labelRevisions = label[1]
1025             if self.verbose:
1026                 print "Change %s is labelled %s" % (change, labelDetails)
1027
1028             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1029                                                     for p in branchPrefixes]))
1030
1031             if len(files) == len(labelRevisions):
1032
1033                 cleanedFiles = {}
1034                 for info in files:
1035                     if info["action"] == "delete":
1036                         continue
1037                     cleanedFiles[info["depotFile"]] = info["rev"]
1038
1039                 if cleanedFiles == labelRevisions:
1040                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1041                     self.gitStream.write("from %s\n" % branch)
1042
1043                     owner = labelDetails["Owner"]
1044                     tagger = ""
1045                     if author in self.users:
1046                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1047                     else:
1048                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1049                     self.gitStream.write("tagger %s\n" % tagger)
1050                     self.gitStream.write("data <<EOT\n")
1051                     self.gitStream.write(labelDetails["Description"])
1052                     self.gitStream.write("EOT\n\n")
1053
1054                 else:
1055                     if not self.silent:
1056                         print ("Tag %s does not match with change %s: files do not match."
1057                                % (labelDetails["label"], change))
1058
1059             else:
1060                 if not self.silent:
1061                     print ("Tag %s does not match with change %s: file count is different."
1062                            % (labelDetails["label"], change))
1063
1064     def getUserCacheFilename(self):
1065         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1066         return home + "/.gitp4-usercache.txt"
1067
1068     def getUserMapFromPerforceServer(self):
1069         if self.userMapFromPerforceServer:
1070             return
1071         self.users = {}
1072
1073         for output in p4CmdList("users"):
1074             if not output.has_key("User"):
1075                 continue
1076             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1077
1078
1079         s = ''
1080         for (key, val) in self.users.items():
1081             s += "%s\t%s\n" % (key, val)
1082
1083         open(self.getUserCacheFilename(), "wb").write(s)
1084         self.userMapFromPerforceServer = True
1085
1086     def loadUserMapFromCache(self):
1087         self.users = {}
1088         self.userMapFromPerforceServer = False
1089         try:
1090             cache = open(self.getUserCacheFilename(), "rb")
1091             lines = cache.readlines()
1092             cache.close()
1093             for line in lines:
1094                 entry = line.strip().split("\t")
1095                 self.users[entry[0]] = entry[1]
1096         except IOError:
1097             self.getUserMapFromPerforceServer()
1098
1099     def getLabels(self):
1100         self.labels = {}
1101
1102         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1103         if len(l) > 0 and not self.silent:
1104             print "Finding files belonging to labels in %s" % `self.depotPaths`
1105
1106         for output in l:
1107             label = output["label"]
1108             revisions = {}
1109             newestChange = 0
1110             if self.verbose:
1111                 print "Querying files for label %s" % label
1112             for file in p4CmdList("files "
1113                                   +  ' '.join (["%s...@%s" % (p, label)
1114                                                 for p in self.depotPaths])):
1115                 revisions[file["depotFile"]] = file["rev"]
1116                 change = int(file["change"])
1117                 if change > newestChange:
1118                     newestChange = change
1119
1120             self.labels[newestChange] = [output, revisions]
1121
1122         if self.verbose:
1123             print "Label changes: %s" % self.labels.keys()
1124
1125     def guessProjectName(self):
1126         for p in self.depotPaths:
1127             if p.endswith("/"):
1128                 p = p[:-1]
1129             p = p[p.strip().rfind("/") + 1:]
1130             if not p.endswith("/"):
1131                p += "/"
1132             return p
1133
1134     def getBranchMapping(self):
1135         lostAndFoundBranches = set()
1136
1137         for info in p4CmdList("branches"):
1138             details = p4Cmd("branch -o %s" % info["branch"])
1139             viewIdx = 0
1140             while details.has_key("View%s" % viewIdx):
1141                 paths = details["View%s" % viewIdx].split(" ")
1142                 viewIdx = viewIdx + 1
1143                 # require standard //depot/foo/... //depot/bar/... mapping
1144                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1145                     continue
1146                 source = paths[0]
1147                 destination = paths[1]
1148                 ## HACK
1149                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1150                     source = source[len(self.depotPaths[0]):-4]
1151                     destination = destination[len(self.depotPaths[0]):-4]
1152
1153                     if destination in self.knownBranches:
1154                         if not self.silent:
1155                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1156                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1157                         continue
1158
1159                     self.knownBranches[destination] = source
1160
1161                     lostAndFoundBranches.discard(destination)
1162
1163                     if source not in self.knownBranches:
1164                         lostAndFoundBranches.add(source)
1165
1166
1167         for branch in lostAndFoundBranches:
1168             self.knownBranches[branch] = branch
1169
1170     def getBranchMappingFromGitBranches(self):
1171         branches = p4BranchesInGit(self.importIntoRemotes)
1172         for branch in branches.keys():
1173             if branch == "master":
1174                 branch = "main"
1175             else:
1176                 branch = branch[len(self.projectName):]
1177             self.knownBranches[branch] = branch
1178
1179     def listExistingP4GitBranches(self):
1180         # branches holds mapping from name to commit
1181         branches = p4BranchesInGit(self.importIntoRemotes)
1182         self.p4BranchesInGit = branches.keys()
1183         for branch in branches.keys():
1184             self.initialParents[self.refPrefix + branch] = branches[branch]
1185
1186     def updateOptionDict(self, d):
1187         option_keys = {}
1188         if self.keepRepoPath:
1189             option_keys['keepRepoPath'] = 1
1190
1191         d["options"] = ' '.join(sorted(option_keys.keys()))
1192
1193     def readOptions(self, d):
1194         self.keepRepoPath = (d.has_key('options')
1195                              and ('keepRepoPath' in d['options']))
1196
1197     def gitRefForBranch(self, branch):
1198         if branch == "main":
1199             return self.refPrefix + "master"
1200
1201         if len(branch) <= 0:
1202             return branch
1203
1204         return self.refPrefix + self.projectName + branch
1205
1206     def gitCommitByP4Change(self, ref, change):
1207         if self.verbose:
1208             print "looking in ref " + ref + " for change %s using bisect..." % change
1209
1210         earliestCommit = ""
1211         latestCommit = parseRevision(ref)
1212
1213         while True:
1214             if self.verbose:
1215                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1216             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1217             if len(next) == 0:
1218                 if self.verbose:
1219                     print "argh"
1220                 return ""
1221             log = extractLogMessageFromGitCommit(next)
1222             settings = extractSettingsGitLog(log)
1223             currentChange = int(settings['change'])
1224             if self.verbose:
1225                 print "current change %s" % currentChange
1226
1227             if currentChange == change:
1228                 if self.verbose:
1229                     print "found %s" % next
1230                 return next
1231
1232             if currentChange < change:
1233                 earliestCommit = "^%s" % next
1234             else:
1235                 latestCommit = "%s" % next
1236
1237         return ""
1238
1239     def importNewBranch(self, branch, maxChange):
1240         # make fast-import flush all changes to disk and update the refs using the checkpoint
1241         # command so that we can try to find the branch parent in the git history
1242         self.gitStream.write("checkpoint\n\n");
1243         self.gitStream.flush();
1244         branchPrefix = self.depotPaths[0] + branch + "/"
1245         range = "@1,%s" % maxChange
1246         #print "prefix" + branchPrefix
1247         changes = p4ChangesForPaths([branchPrefix], range)
1248         if len(changes) <= 0:
1249             return False
1250         firstChange = changes[0]
1251         #print "first change in branch: %s" % firstChange
1252         sourceBranch = self.knownBranches[branch]
1253         sourceDepotPath = self.depotPaths[0] + sourceBranch
1254         sourceRef = self.gitRefForBranch(sourceBranch)
1255         #print "source " + sourceBranch
1256
1257         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1258         #print "branch parent: %s" % branchParentChange
1259         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1260         if len(gitParent) > 0:
1261             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1262             #print "parent git commit: %s" % gitParent
1263
1264         self.importChanges(changes)
1265         return True
1266
1267     def importChanges(self, changes):
1268         cnt = 1
1269         for change in changes:
1270             description = p4Cmd("describe %s" % change)
1271             self.updateOptionDict(description)
1272
1273             if not self.silent:
1274                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1275                 sys.stdout.flush()
1276             cnt = cnt + 1
1277
1278             try:
1279                 if self.detectBranches:
1280                     branches = self.splitFilesIntoBranches(description)
1281                     for branch in branches.keys():
1282                         ## HACK  --hwn
1283                         branchPrefix = self.depotPaths[0] + branch + "/"
1284
1285                         parent = ""
1286
1287                         filesForCommit = branches[branch]
1288
1289                         if self.verbose:
1290                             print "branch is %s" % branch
1291
1292                         self.updatedBranches.add(branch)
1293
1294                         if branch not in self.createdBranches:
1295                             self.createdBranches.add(branch)
1296                             parent = self.knownBranches[branch]
1297                             if parent == branch:
1298                                 parent = ""
1299                             else:
1300                                 fullBranch = self.projectName + branch
1301                                 if fullBranch not in self.p4BranchesInGit:
1302                                     if not self.silent:
1303                                         print("\n    Importing new branch %s" % fullBranch);
1304                                     if self.importNewBranch(branch, change - 1):
1305                                         parent = ""
1306                                         self.p4BranchesInGit.append(fullBranch)
1307                                     if not self.silent:
1308                                         print("\n    Resuming with change %s" % change);
1309
1310                                 if self.verbose:
1311                                     print "parent determined through known branches: %s" % parent
1312
1313                         branch = self.gitRefForBranch(branch)
1314                         parent = self.gitRefForBranch(parent)
1315
1316                         if self.verbose:
1317                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1318
1319                         if len(parent) == 0 and branch in self.initialParents:
1320                             parent = self.initialParents[branch]
1321                             del self.initialParents[branch]
1322
1323                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1324                 else:
1325                     files = self.extractFilesFromCommit(description)
1326                     self.commit(description, files, self.branch, self.depotPaths,
1327                                 self.initialParent)
1328                     self.initialParent = ""
1329             except IOError:
1330                 print self.gitError.read()
1331                 sys.exit(1)
1332
1333     def importHeadRevision(self, revision):
1334         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1335
1336         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1337         details["desc"] = ("Initial import of %s from the state at revision %s"
1338                            % (' '.join(self.depotPaths), revision))
1339         details["change"] = revision
1340         newestRevision = 0
1341
1342         fileCnt = 0
1343         for info in p4CmdList("files "
1344                               +  ' '.join(["%s...%s"
1345                                            % (p, revision)
1346                                            for p in self.depotPaths])):
1347
1348             if info['code'] == 'error':
1349                 sys.stderr.write("p4 returned an error: %s\n"
1350                                  % info['data'])
1351                 sys.exit(1)
1352
1353
1354             change = int(info["change"])
1355             if change > newestRevision:
1356                 newestRevision = change
1357
1358             if info["action"] == "delete":
1359                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1360                 #fileCnt = fileCnt + 1
1361                 continue
1362
1363             for prop in ["depotFile", "rev", "action", "type" ]:
1364                 details["%s%s" % (prop, fileCnt)] = info[prop]
1365
1366             fileCnt = fileCnt + 1
1367
1368         details["change"] = newestRevision
1369         self.updateOptionDict(details)
1370         try:
1371             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1372         except IOError:
1373             print "IO error with git fast-import. Is your git version recent enough?"
1374             print self.gitError.read()
1375
1376
1377     def getClientSpec(self):
1378         specList = p4CmdList( "client -o" )
1379         temp = {}
1380         for entry in specList:
1381             for k,v in entry.iteritems():
1382                 if k.startswith("View"):
1383                     if v.startswith('"'):
1384                         start = 1
1385                     else:
1386                         start = 0
1387                     index = v.find("...")
1388                     v = v[start:index]
1389                     if v.startswith("-"):
1390                         v = v[1:]
1391                         temp[v] = -len(v)
1392                     else:
1393                         temp[v] = len(v)
1394         self.clientSpecDirs = temp.items()
1395         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1396
1397     def run(self, args):
1398         self.depotPaths = []
1399         self.changeRange = ""
1400         self.initialParent = ""
1401         self.previousDepotPaths = []
1402
1403         # map from branch depot path to parent branch
1404         self.knownBranches = {}
1405         self.initialParents = {}
1406         self.hasOrigin = originP4BranchesExist()
1407         if not self.syncWithOrigin:
1408             self.hasOrigin = False
1409
1410         if self.importIntoRemotes:
1411             self.refPrefix = "refs/remotes/p4/"
1412         else:
1413             self.refPrefix = "refs/heads/p4/"
1414
1415         if self.syncWithOrigin and self.hasOrigin:
1416             if not self.silent:
1417                 print "Syncing with origin first by calling git fetch origin"
1418             system("git fetch origin")
1419
1420         if len(self.branch) == 0:
1421             self.branch = self.refPrefix + "master"
1422             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1423                 system("git update-ref %s refs/heads/p4" % self.branch)
1424                 system("git branch -D p4");
1425             # create it /after/ importing, when master exists
1426             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1427                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1428
1429         if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1430             self.getClientSpec()
1431
1432         # TODO: should always look at previous commits,
1433         # merge with previous imports, if possible.
1434         if args == []:
1435             if self.hasOrigin:
1436                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1437             self.listExistingP4GitBranches()
1438
1439             if len(self.p4BranchesInGit) > 1:
1440                 if not self.silent:
1441                     print "Importing from/into multiple branches"
1442                 self.detectBranches = True
1443
1444             if self.verbose:
1445                 print "branches: %s" % self.p4BranchesInGit
1446
1447             p4Change = 0
1448             for branch in self.p4BranchesInGit:
1449                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1450
1451                 settings = extractSettingsGitLog(logMsg)
1452
1453                 self.readOptions(settings)
1454                 if (settings.has_key('depot-paths')
1455                     and settings.has_key ('change')):
1456                     change = int(settings['change']) + 1
1457                     p4Change = max(p4Change, change)
1458
1459                     depotPaths = sorted(settings['depot-paths'])
1460                     if self.previousDepotPaths == []:
1461                         self.previousDepotPaths = depotPaths
1462                     else:
1463                         paths = []
1464                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1465                             for i in range(0, min(len(cur), len(prev))):
1466                                 if cur[i] <> prev[i]:
1467                                     i = i - 1
1468                                     break
1469
1470                             paths.append (cur[:i + 1])
1471
1472                         self.previousDepotPaths = paths
1473
1474             if p4Change > 0:
1475                 self.depotPaths = sorted(self.previousDepotPaths)
1476                 self.changeRange = "@%s,#head" % p4Change
1477                 if not self.detectBranches:
1478                     self.initialParent = parseRevision(self.branch)
1479                 if not self.silent and not self.detectBranches:
1480                     print "Performing incremental import into %s git branch" % self.branch
1481
1482         if not self.branch.startswith("refs/"):
1483             self.branch = "refs/heads/" + self.branch
1484
1485         if len(args) == 0 and self.depotPaths:
1486             if not self.silent:
1487                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1488         else:
1489             if self.depotPaths and self.depotPaths != args:
1490                 print ("previous import used depot path %s and now %s was specified. "
1491                        "This doesn't work!" % (' '.join (self.depotPaths),
1492                                                ' '.join (args)))
1493                 sys.exit(1)
1494
1495             self.depotPaths = sorted(args)
1496
1497         revision = ""
1498         self.users = {}
1499
1500         newPaths = []
1501         for p in self.depotPaths:
1502             if p.find("@") != -1:
1503                 atIdx = p.index("@")
1504                 self.changeRange = p[atIdx:]
1505                 if self.changeRange == "@all":
1506                     self.changeRange = ""
1507                 elif ',' not in self.changeRange:
1508                     revision = self.changeRange
1509                     self.changeRange = ""
1510                 p = p[:atIdx]
1511             elif p.find("#") != -1:
1512                 hashIdx = p.index("#")
1513                 revision = p[hashIdx:]
1514                 p = p[:hashIdx]
1515             elif self.previousDepotPaths == []:
1516                 revision = "#head"
1517
1518             p = re.sub ("\.\.\.$", "", p)
1519             if not p.endswith("/"):
1520                 p += "/"
1521
1522             newPaths.append(p)
1523
1524         self.depotPaths = newPaths
1525
1526
1527         self.loadUserMapFromCache()
1528         self.labels = {}
1529         if self.detectLabels:
1530             self.getLabels();
1531
1532         if self.detectBranches:
1533             ## FIXME - what's a P4 projectName ?
1534             self.projectName = self.guessProjectName()
1535
1536             if self.hasOrigin:
1537                 self.getBranchMappingFromGitBranches()
1538             else:
1539                 self.getBranchMapping()
1540             if self.verbose:
1541                 print "p4-git branches: %s" % self.p4BranchesInGit
1542                 print "initial parents: %s" % self.initialParents
1543             for b in self.p4BranchesInGit:
1544                 if b != "master":
1545
1546                     ## FIXME
1547                     b = b[len(self.projectName):]
1548                 self.createdBranches.add(b)
1549
1550         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1551
1552         importProcess = subprocess.Popen(["git", "fast-import"],
1553                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1554                                          stderr=subprocess.PIPE);
1555         self.gitOutput = importProcess.stdout
1556         self.gitStream = importProcess.stdin
1557         self.gitError = importProcess.stderr
1558
1559         if revision:
1560             self.importHeadRevision(revision)
1561         else:
1562             changes = []
1563
1564             if len(self.changesFile) > 0:
1565                 output = open(self.changesFile).readlines()
1566                 changeSet = Set()
1567                 for line in output:
1568                     changeSet.add(int(line))
1569
1570                 for change in changeSet:
1571                     changes.append(change)
1572
1573                 changes.sort()
1574             else:
1575                 if self.verbose:
1576                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1577                                                               self.changeRange)
1578                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1579
1580                 if len(self.maxChanges) > 0:
1581                     changes = changes[:min(int(self.maxChanges), len(changes))]
1582
1583             if len(changes) == 0:
1584                 if not self.silent:
1585                     print "No changes to import!"
1586                 return True
1587
1588             if not self.silent and not self.detectBranches:
1589                 print "Import destination: %s" % self.branch
1590
1591             self.updatedBranches = set()
1592
1593             self.importChanges(changes)
1594
1595             if not self.silent:
1596                 print ""
1597                 if len(self.updatedBranches) > 0:
1598                     sys.stdout.write("Updated branches: ")
1599                     for b in self.updatedBranches:
1600                         sys.stdout.write("%s " % b)
1601                     sys.stdout.write("\n")
1602
1603         self.gitStream.close()
1604         if importProcess.wait() != 0:
1605             die("fast-import failed: %s" % self.gitError.read())
1606         self.gitOutput.close()
1607         self.gitError.close()
1608
1609         return True
1610
1611 class P4Rebase(Command):
1612     def __init__(self):
1613         Command.__init__(self)
1614         self.options = [ ]
1615         self.description = ("Fetches the latest revision from perforce and "
1616                             + "rebases the current work (branch) against it")
1617         self.verbose = False
1618
1619     def run(self, args):
1620         sync = P4Sync()
1621         sync.run([])
1622
1623         return self.rebase()
1624
1625     def rebase(self):
1626         if os.system("git update-index --refresh") != 0:
1627             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1628         if len(read_pipe("git diff-index HEAD --")) > 0:
1629             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1630
1631         [upstream, settings] = findUpstreamBranchPoint()
1632         if len(upstream) == 0:
1633             die("Cannot find upstream branchpoint for rebase")
1634
1635         # the branchpoint may be p4/foo~3, so strip off the parent
1636         upstream = re.sub("~[0-9]+$", "", upstream)
1637
1638         print "Rebasing the current branch onto %s" % upstream
1639         oldHead = read_pipe("git rev-parse HEAD").strip()
1640         system("git rebase %s" % upstream)
1641         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1642         return True
1643
1644 class P4Clone(P4Sync):
1645     def __init__(self):
1646         P4Sync.__init__(self)
1647         self.description = "Creates a new git repository and imports from Perforce into it"
1648         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1649         self.options += [
1650             optparse.make_option("--destination", dest="cloneDestination",
1651                                  action='store', default=None,
1652                                  help="where to leave result of the clone"),
1653             optparse.make_option("-/", dest="cloneExclude",
1654                                  action="append", type="string",
1655                                  help="exclude depot path")
1656         ]
1657         self.cloneDestination = None
1658         self.needsGit = False
1659
1660     # This is required for the "append" cloneExclude action
1661     def ensure_value(self, attr, value):
1662         if not hasattr(self, attr) or getattr(self, attr) is None:
1663             setattr(self, attr, value)
1664         return getattr(self, attr)
1665
1666     def defaultDestination(self, args):
1667         ## TODO: use common prefix of args?
1668         depotPath = args[0]
1669         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1670         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1671         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1672         depotDir = re.sub(r"/$", "", depotDir)
1673         return os.path.split(depotDir)[1]
1674
1675     def run(self, args):
1676         if len(args) < 1:
1677             return False
1678
1679         if self.keepRepoPath and not self.cloneDestination:
1680             sys.stderr.write("Must specify destination for --keep-path\n")
1681             sys.exit(1)
1682
1683         depotPaths = args
1684
1685         if not self.cloneDestination and len(depotPaths) > 1:
1686             self.cloneDestination = depotPaths[-1]
1687             depotPaths = depotPaths[:-1]
1688
1689         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1690         for p in depotPaths:
1691             if not p.startswith("//"):
1692                 return False
1693
1694         if not self.cloneDestination:
1695             self.cloneDestination = self.defaultDestination(args)
1696
1697         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1698         if not os.path.exists(self.cloneDestination):
1699             os.makedirs(self.cloneDestination)
1700         os.chdir(self.cloneDestination)
1701         system("git init")
1702         self.gitdir = os.getcwd() + "/.git"
1703         if not P4Sync.run(self, depotPaths):
1704             return False
1705         if self.branch != "master":
1706             if gitBranchExists("refs/remotes/p4/master"):
1707                 system("git branch master refs/remotes/p4/master")
1708                 system("git checkout -f")
1709             else:
1710                 print "Could not detect main branch. No checkout/master branch created."
1711
1712         return True
1713
1714 class P4Branches(Command):
1715     def __init__(self):
1716         Command.__init__(self)
1717         self.options = [ ]
1718         self.description = ("Shows the git branches that hold imports and their "
1719                             + "corresponding perforce depot paths")
1720         self.verbose = False
1721
1722     def run(self, args):
1723         if originP4BranchesExist():
1724             createOrUpdateBranchesFromOrigin()
1725
1726         cmdline = "git rev-parse --symbolic "
1727         cmdline += " --remotes"
1728
1729         for line in read_pipe_lines(cmdline):
1730             line = line.strip()
1731
1732             if not line.startswith('p4/') or line == "p4/HEAD":
1733                 continue
1734             branch = line
1735
1736             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1737             settings = extractSettingsGitLog(log)
1738
1739             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1740         return True
1741
1742 class HelpFormatter(optparse.IndentedHelpFormatter):
1743     def __init__(self):
1744         optparse.IndentedHelpFormatter.__init__(self)
1745
1746     def format_description(self, description):
1747         if description:
1748             return description + "\n"
1749         else:
1750             return ""
1751
1752 def printUsage(commands):
1753     print "usage: %s <command> [options]" % sys.argv[0]
1754     print ""
1755     print "valid commands: %s" % ", ".join(commands)
1756     print ""
1757     print "Try %s <command> --help for command specific help." % sys.argv[0]
1758     print ""
1759
1760 commands = {
1761     "debug" : P4Debug,
1762     "submit" : P4Submit,
1763     "commit" : P4Submit,
1764     "sync" : P4Sync,
1765     "rebase" : P4Rebase,
1766     "clone" : P4Clone,
1767     "rollback" : P4RollBack,
1768     "branches" : P4Branches
1769 }
1770
1771
1772 def main():
1773     if len(sys.argv[1:]) == 0:
1774         printUsage(commands.keys())
1775         sys.exit(2)
1776
1777     cmd = ""
1778     cmdName = sys.argv[1]
1779     try:
1780         klass = commands[cmdName]
1781         cmd = klass()
1782     except KeyError:
1783         print "unknown command %s" % cmdName
1784         print ""
1785         printUsage(commands.keys())
1786         sys.exit(2)
1787
1788     options = cmd.options
1789     cmd.gitdir = os.environ.get("GIT_DIR", None)
1790
1791     args = sys.argv[2:]
1792
1793     if len(options) > 0:
1794         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1795
1796         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1797                                        options,
1798                                        description = cmd.description,
1799                                        formatter = HelpFormatter())
1800
1801         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1802     global verbose
1803     verbose = cmd.verbose
1804     if cmd.needsGit:
1805         if cmd.gitdir == None:
1806             cmd.gitdir = os.path.abspath(".git")
1807             if not isValidGitDir(cmd.gitdir):
1808                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1809                 if os.path.exists(cmd.gitdir):
1810                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1811                     if len(cdup) > 0:
1812                         os.chdir(cdup);
1813
1814         if not isValidGitDir(cmd.gitdir):
1815             if isValidGitDir(cmd.gitdir + "/.git"):
1816                 cmd.gitdir += "/.git"
1817             else:
1818                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1819
1820         os.environ["GIT_DIR"] = cmd.gitdir
1821
1822     if not cmd.run(args):
1823         parser.print_help()
1824
1825
1826 if __name__ == '__main__':
1827     main()