contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     if os.name == 'nt':
  56         os.environ['PWD']=dir
  57     os.chdir(dir)
  58
  59 def die(msg):
  60     if verbose:
  61         raise Exception(msg)
  62     else:
  63         sys.stderr.write(msg + "\n")
  64         sys.exit(1)
  65
  66 def write_pipe(c, str):
  67     if verbose:
  68         sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70     pipe = os.popen(c, 'w')
  71     val = pipe.write(str)
  72     if pipe.close():
  73         die('Command failed: %s' % c)
  74
  75     return val
  76
  77 def p4_write_pipe(c, str):
  78     real_cmd = p4_build_cmd(c)
  79     return write_pipe(real_cmd, str)
  80
  81 def read_pipe(c, ignore_error=False):
  82     if verbose:
  83         sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85     pipe = os.popen(c, 'rb')
  86     val = pipe.read()
  87     if pipe.close() and not ignore_error:
  88         die('Command failed: %s' % c)
  89
  90     return val
  91
  92 def p4_read_pipe(c, ignore_error=False):
  93     real_cmd = p4_build_cmd(c)
  94     return read_pipe(real_cmd, ignore_error)
  95
  96 def read_pipe_lines(c):
  97     if verbose:
  98         sys.stderr.write('Reading pipe: %s\n' % c)
  99     ## todo: check return status
 100     pipe = os.popen(c, 'rb')
 101     val = pipe.readlines()
 102     if pipe.close():
 103         die('Command failed: %s' % c)
 104
 105     return val
 106
 107 def p4_read_pipe_lines(c):
 108     """Specifically invoke p4 on the command supplied. """
 109     real_cmd = p4_build_cmd(c)
 110     return read_pipe_lines(real_cmd)
 111
 112 def system(cmd):
 113     if verbose:
 114         sys.stderr.write("executing %s\n" % cmd)
 115     if os.system(cmd) != 0:
 116         die("command failed: %s" % cmd)
 117
 118 def p4_system(cmd):
 119     """Specifically invoke p4 as the system command. """
 120     real_cmd = p4_build_cmd(cmd)
 121     return system(real_cmd)
 122
 123 def isP4Exec(kind):
 124     """Determine if a Perforce 'kind' should have execute permission
 125
 126     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128     a plus sign, it is also executable"""
 129     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131 def setP4ExecBit(file, mode):
 132     # Reopens an already open file and changes the execute bit to match
 133     # the execute bit setting in the passed in mode.
 134
 135     p4Type = "+x"
 136
 137     if not isModeExec(mode):
 138         p4Type = getP4OpenedType(file)
 139         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141         if p4Type[-1] == "+":
 142             p4Type = p4Type[0:-1]
 143
 144     p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146 def getP4OpenedType(file):
 147     # Returns the perforce file type for the given file.
 148
 149     result = p4_read_pipe("opened %s" % file)
 150     match = re.match(".*\((.+)\)\r?$", result)
 151     if match:
 152         return match.group(1)
 153     else:
 154         die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156 def diffTreePattern():
 157     # This is a simple generator for the diff tree regex pattern. This could be
 158     # a class variable if this and parseDiffTreeEntry were a part of a class.
 159     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160     while True:
 161         yield pattern
 162
 163 def parseDiffTreeEntry(entry):
 164     """Parses a single diff tree entry into its component elements.
 165
 166     See git-diff-tree(1) manpage for details about the format of the diff
 167     output. This method returns a dictionary with the following elements:
 168
 169     src_mode - The mode of the source file
 170     dst_mode - The mode of the destination file
 171     src_sha1 - The sha1 for the source file
 172     dst_sha1 - The sha1 fr the destination file
 173     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174     status_score - The score for the status (applicable for 'C' and 'R'
 175                    statuses). This is None if there is no score.
 176     src - The path for the source file.
 177     dst - The path for the destination file. This is only present for
 178           copy or renames. If it is not present, this is None.
 179
 180     If the pattern is not matched, None is returned."""
 181
 182     match = diffTreePattern().next().match(entry)
 183     if match:
 184         return {
 185             'src_mode': match.group(1),
 186             'dst_mode': match.group(2),
 187             'src_sha1': match.group(3),
 188             'dst_sha1': match.group(4),
 189             'status': match.group(5),
 190             'status_score': match.group(6),
 191             'src': match.group(7),
 192             'dst': match.group(10)
 193         }
 194     return None
 195
 196 def isModeExec(mode):
 197     # Returns True if the given git mode represents an executable file,
 198     # otherwise False.
 199     return mode[-3:] == "755"
 200
 201 def isModeExecChanged(src_mode, dst_mode):
 202     return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 205     cmd = p4_build_cmd("-G %s" % (cmd))
 206     if verbose:
 207         sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209     # Use a temporary file to avoid deadlocks without
 210     # subprocess.communicate(), which would put another copy
 211     # of stdout into memory.
 212     stdin_file = None
 213     if stdin is not None:
 214         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215         stdin_file.write(stdin)
 216         stdin_file.flush()
 217         stdin_file.seek(0)
 218
 219     p4 = subprocess.Popen(cmd, shell=True,
 220                           stdin=stdin_file,
 221                           stdout=subprocess.PIPE)
 222
 223     result = []
 224     try:
 225         while True:
 226             entry = marshal.load(p4.stdout)
 227             if cb is not None:
 228                 cb(entry)
 229             else:
 230                 result.append(entry)
 231     except EOFError:
 232         pass
 233     exitCode = p4.wait()
 234     if exitCode != 0:
 235         entry = {}
 236         entry["p4ExitCode"] = exitCode
 237         result.append(entry)
 238
 239     return result
 240
 241 def p4Cmd(cmd):
 242     list = p4CmdList(cmd)
 243     result = {}
 244     for entry in list:
 245         result.update(entry)
 246     return result;
 247
 248 def p4Where(depotPath):
 249     if not depotPath.endswith("/"):
 250         depotPath += "/"
 251     depotPath = depotPath + "..."
 252     outputList = p4CmdList("where %s" % depotPath)
 253     output = None
 254     for entry in outputList:
 255         if "depotFile" in entry:
 256             if entry["depotFile"] == depotPath:
 257                 output = entry
 258                 break
 259         elif "data" in entry:
 260             data = entry.get("data")
 261             space = data.find(" ")
 262             if data[:space] == depotPath:
 263                 output = entry
 264                 break
 265     if output == None:
 266         return ""
 267     if output["code"] == "error":
 268         return ""
 269     clientPath = ""
 270     if "path" in output:
 271         clientPath = output.get("path")
 272     elif "data" in output:
 273         data = output.get("data")
 274         lastSpace = data.rfind(" ")
 275         clientPath = data[lastSpace + 1:]
 276
 277     if clientPath.endswith("..."):
 278         clientPath = clientPath[:-3]
 279     return clientPath
 280
 281 def currentGitBranch():
 282     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 283
 284 def isValidGitDir(path):
 285     if (os.path.exists(path + "/HEAD")
 286         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 287         return True;
 288     return False
 289
 290 def parseRevision(ref):
 291     return read_pipe("git rev-parse %s" % ref).strip()
 292
 293 def extractLogMessageFromGitCommit(commit):
 294     logMessage = ""
 295
 296     ## fixme: title is first line of commit, not 1st paragraph.
 297     foundTitle = False
 298     for log in read_pipe_lines("git cat-file commit %s" % commit):
 299        if not foundTitle:
 300            if len(log) == 1:
 301                foundTitle = True
 302            continue
 303
 304        logMessage += log
 305     return logMessage
 306
 307 def extractSettingsGitLog(log):
 308     values = {}
 309     for line in log.split("\n"):
 310         line = line.strip()
 311         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 312         if not m:
 313             continue
 314
 315         assignments = m.group(1).split (':')
 316         for a in assignments:
 317             vals = a.split ('=')
 318             key = vals[0].strip()
 319             val = ('='.join (vals[1:])).strip()
 320             if val.endswith ('\"') and val.startswith('"'):
 321                 val = val[1:-1]
 322
 323             values[key] = val
 324
 325     paths = values.get("depot-paths")
 326     if not paths:
 327         paths = values.get("depot-path")
 328     if paths:
 329         values['depot-paths'] = paths.split(',')
 330     return values
 331
 332 def gitBranchExists(branch):
 333     proc = subprocess.Popen(["git", "rev-parse", branch],
 334                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 335     return proc.wait() == 0;
 336
 337 _gitConfig = {}
 338 def gitConfig(key):
 339     if not _gitConfig.has_key(key):
 340         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 341     return _gitConfig[key]
 342
 343 def p4BranchesInGit(branchesAreInRemotes = True):
 344     branches = {}
 345
 346     cmdline = "git rev-parse --symbolic "
 347     if branchesAreInRemotes:
 348         cmdline += " --remotes"
 349     else:
 350         cmdline += " --branches"
 351
 352     for line in read_pipe_lines(cmdline):
 353         line = line.strip()
 354
 355         ## only import to p4/
 356         if not line.startswith('p4/') or line == "p4/HEAD":
 357             continue
 358         branch = line
 359
 360         # strip off p4
 361         branch = re.sub ("^p4/", "", line)
 362
 363         branches[branch] = parseRevision(line)
 364     return branches
 365
 366 def findUpstreamBranchPoint(head = "HEAD"):
 367     branches = p4BranchesInGit()
 368     # map from depot-path to branch name
 369     branchByDepotPath = {}
 370     for branch in branches.keys():
 371         tip = branches[branch]
 372         log = extractLogMessageFromGitCommit(tip)
 373         settings = extractSettingsGitLog(log)
 374         if settings.has_key("depot-paths"):
 375             paths = ",".join(settings["depot-paths"])
 376             branchByDepotPath[paths] = "remotes/p4/" + branch
 377
 378     settings = None
 379     parent = 0
 380     while parent < 65535:
 381         commit = head + "~%s" % parent
 382         log = extractLogMessageFromGitCommit(commit)
 383         settings = extractSettingsGitLog(log)
 384         if settings.has_key("depot-paths"):
 385             paths = ",".join(settings["depot-paths"])
 386             if branchByDepotPath.has_key(paths):
 387                 return [branchByDepotPath[paths], settings]
 388
 389         parent = parent + 1
 390
 391     return ["", settings]
 392
 393 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 394     if not silent:
 395         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 396                % localRefPrefix)
 397
 398     originPrefix = "origin/p4/"
 399
 400     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 401         line = line.strip()
 402         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 403             continue
 404
 405         headName = line[len(originPrefix):]
 406         remoteHead = localRefPrefix + headName
 407         originHead = line
 408
 409         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 410         if (not original.has_key('depot-paths')
 411             or not original.has_key('change')):
 412             continue
 413
 414         update = False
 415         if not gitBranchExists(remoteHead):
 416             if verbose:
 417                 print "creating %s" % remoteHead
 418             update = True
 419         else:
 420             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 421             if settings.has_key('change') > 0:
 422                 if settings['depot-paths'] == original['depot-paths']:
 423                     originP4Change = int(original['change'])
 424                     p4Change = int(settings['change'])
 425                     if originP4Change > p4Change:
 426                         print ("%s (%s) is newer than %s (%s). "
 427                                "Updating p4 branch from origin."
 428                                % (originHead, originP4Change,
 429                                   remoteHead, p4Change))
 430                         update = True
 431                 else:
 432                     print ("Ignoring: %s was imported from %s while "
 433                            "%s was imported from %s"
 434                            % (originHead, ','.join(original['depot-paths']),
 435                               remoteHead, ','.join(settings['depot-paths'])))
 436
 437         if update:
 438             system("git update-ref %s %s" % (remoteHead, originHead))
 439
 440 def originP4BranchesExist():
 441         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 442
 443 def p4ChangesForPaths(depotPaths, changeRange):
 444     assert depotPaths
 445     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 446                                                         for p in depotPaths]))
 447
 448     changes = {}
 449     for line in output:
 450         changeNum = int(line.split(" ")[1])
 451         changes[changeNum] = True
 452
 453     changelist = changes.keys()
 454     changelist.sort()
 455     return changelist
 456
 457 class Command:
 458     def __init__(self):
 459         self.usage = "usage: %prog [options]"
 460         self.needsGit = True
 461
 462 class P4Debug(Command):
 463     def __init__(self):
 464         Command.__init__(self)
 465         self.options = [
 466             optparse.make_option("--verbose", dest="verbose", action="store_true",
 467                                  default=False),
 468             ]
 469         self.description = "A tool to debug the output of p4 -G."
 470         self.needsGit = False
 471         self.verbose = False
 472
 473     def run(self, args):
 474         j = 0
 475         for output in p4CmdList(" ".join(args)):
 476             print 'Element: %d' % j
 477             j += 1
 478             print output
 479         return True
 480
 481 class P4RollBack(Command):
 482     def __init__(self):
 483         Command.__init__(self)
 484         self.options = [
 485             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 486             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 487         ]
 488         self.description = "A tool to debug the multi-branch import. Don't use :)"
 489         self.verbose = False
 490         self.rollbackLocalBranches = False
 491
 492     def run(self, args):
 493         if len(args) != 1:
 494             return False
 495         maxChange = int(args[0])
 496
 497         if "p4ExitCode" in p4Cmd("changes -m 1"):
 498             die("Problems executing p4");
 499
 500         if self.rollbackLocalBranches:
 501             refPrefix = "refs/heads/"
 502             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 503         else:
 504             refPrefix = "refs/remotes/"
 505             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 506
 507         for line in lines:
 508             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 509                 line = line.strip()
 510                 ref = refPrefix + line
 511                 log = extractLogMessageFromGitCommit(ref)
 512                 settings = extractSettingsGitLog(log)
 513
 514                 depotPaths = settings['depot-paths']
 515                 change = settings['change']
 516
 517                 changed = False
 518
 519                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 520                                                            for p in depotPaths]))) == 0:
 521                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 522                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 523                     continue
 524
 525                 while change and int(change) > maxChange:
 526                     changed = True
 527                     if self.verbose:
 528                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 529                     system("git update-ref %s \"%s^\"" % (ref, ref))
 530                     log = extractLogMessageFromGitCommit(ref)
 531                     settings =  extractSettingsGitLog(log)
 532
 533
 534                     depotPaths = settings['depot-paths']
 535                     change = settings['change']
 536
 537                 if changed:
 538                     print "%s rewound to %s" % (ref, change)
 539
 540         return True
 541
 542 class P4Submit(Command):
 543     def __init__(self):
 544         Command.__init__(self)
 545         self.options = [
 546                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 547                 optparse.make_option("--origin", dest="origin"),
 548                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 549         ]
 550         self.description = "Submit changes from git to the perforce depot."
 551         self.usage += " [name of git branch to submit into perforce depot]"
 552         self.interactive = True
 553         self.origin = ""
 554         self.detectRename = False
 555         self.verbose = False
 556         self.isWindows = (platform.system() == "Windows")
 557
 558     def check(self):
 559         if len(p4CmdList("opened ...")) > 0:
 560             die("You have files opened with perforce! Close them before starting the sync.")
 561
 562     # replaces everything between 'Description:' and the next P4 submit template field with the
 563     # commit message
 564     def prepareLogMessage(self, template, message):
 565         result = ""
 566
 567         inDescriptionSection = False
 568
 569         for line in template.split("\n"):
 570             if line.startswith("#"):
 571                 result += line + "\n"
 572                 continue
 573
 574             if inDescriptionSection:
 575                 if line.startswith("Files:"):
 576                     inDescriptionSection = False
 577                 else:
 578                     continue
 579             else:
 580                 if line.startswith("Description:"):
 581                     inDescriptionSection = True
 582                     line += "\n"
 583                     for messageLine in message.split("\n"):
 584                         line += "\t" + messageLine + "\n"
 585
 586             result += line + "\n"
 587
 588         return result
 589
 590     def prepareSubmitTemplate(self):
 591         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 592         template = ""
 593         inFilesSection = False
 594         for line in p4_read_pipe_lines("change -o"):
 595             if line.endswith("\r\n"):
 596                 line = line[:-2] + "\n"
 597             if inFilesSection:
 598                 if line.startswith("\t"):
 599                     # path starts and ends with a tab
 600                     path = line[1:]
 601                     lastTab = path.rfind("\t")
 602                     if lastTab != -1:
 603                         path = path[:lastTab]
 604                         if not path.startswith(self.depotPath):
 605                             continue
 606                 else:
 607                     inFilesSection = False
 608             else:
 609                 if line.startswith("Files:"):
 610                     inFilesSection = True
 611
 612             template += line
 613
 614         return template
 615
 616     def applyCommit(self, id):
 617         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 618         diffOpts = ("", "-M")[self.detectRename]
 619         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 620         filesToAdd = set()
 621         filesToDelete = set()
 622         editedFiles = set()
 623         filesToChangeExecBit = {}
 624         for line in diff:
 625             diff = parseDiffTreeEntry(line)
 626             modifier = diff['status']
 627             path = diff['src']
 628             if modifier == "M":
 629                 p4_system("edit \"%s\"" % path)
 630                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 631                     filesToChangeExecBit[path] = diff['dst_mode']
 632                 editedFiles.add(path)
 633             elif modifier == "A":
 634                 filesToAdd.add(path)
 635                 filesToChangeExecBit[path] = diff['dst_mode']
 636                 if path in filesToDelete:
 637                     filesToDelete.remove(path)
 638             elif modifier == "D":
 639                 filesToDelete.add(path)
 640                 if path in filesToAdd:
 641                     filesToAdd.remove(path)
 642             elif modifier == "R":
 643                 src, dest = diff['src'], diff['dst']
 644                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 645                 p4_system("edit \"%s\"" % (dest))
 646                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 647                     filesToChangeExecBit[dest] = diff['dst_mode']
 648                 os.unlink(dest)
 649                 editedFiles.add(dest)
 650                 filesToDelete.add(src)
 651             else:
 652                 die("unknown modifier %s for %s" % (modifier, path))
 653
 654         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 655         patchcmd = diffcmd + " | git apply "
 656         tryPatchCmd = patchcmd + "--check -"
 657         applyPatchCmd = patchcmd + "--check --apply -"
 658
 659         if os.system(tryPatchCmd) != 0:
 660             print "Unfortunately applying the change failed!"
 661             print "What do you want to do?"
 662             response = "x"
 663             while response != "s" and response != "a" and response != "w":
 664                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 665                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 666             if response == "s":
 667                 print "Skipping! Good luck with the next patches..."
 668                 for f in editedFiles:
 669                     p4_system("revert \"%s\"" % f);
 670                 for f in filesToAdd:
 671                     system("rm %s" %f)
 672                 return
 673             elif response == "a":
 674                 os.system(applyPatchCmd)
 675                 if len(filesToAdd) > 0:
 676                     print "You may also want to call p4 add on the following files:"
 677                     print " ".join(filesToAdd)
 678                 if len(filesToDelete):
 679                     print "The following files should be scheduled for deletion with p4 delete:"
 680                     print " ".join(filesToDelete)
 681                 die("Please resolve and submit the conflict manually and "
 682                     + "continue afterwards with git-p4 submit --continue")
 683             elif response == "w":
 684                 system(diffcmd + " > patch.txt")
 685                 print "Patch saved to patch.txt in %s !" % self.clientPath
 686                 die("Please resolve and submit the conflict manually and "
 687                     "continue afterwards with git-p4 submit --continue")
 688
 689         system(applyPatchCmd)
 690
 691         for f in filesToAdd:
 692             p4_system("add \"%s\"" % f)
 693         for f in filesToDelete:
 694             p4_system("revert \"%s\"" % f)
 695             p4_system("delete \"%s\"" % f)
 696
 697         # Set/clear executable bits
 698         for f in filesToChangeExecBit.keys():
 699             mode = filesToChangeExecBit[f]
 700             setP4ExecBit(f, mode)
 701
 702         logMessage = extractLogMessageFromGitCommit(id)
 703         logMessage = logMessage.strip()
 704
 705         template = self.prepareSubmitTemplate()
 706
 707         if self.interactive:
 708             submitTemplate = self.prepareLogMessage(template, logMessage)
 709             if os.environ.has_key("P4DIFF"):
 710                 del(os.environ["P4DIFF"])
 711             diff = p4_read_pipe("diff -du ...")
 712
 713             newdiff = ""
 714             for newFile in filesToAdd:
 715                 newdiff += "==== new file ====\n"
 716                 newdiff += "--- /dev/null\n"
 717                 newdiff += "+++ %s\n" % newFile
 718                 f = open(newFile, "r")
 719                 for line in f.readlines():
 720                     newdiff += "+" + line
 721                 f.close()
 722
 723             separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725             [handle, fileName] = tempfile.mkstemp()
 726             tmpFile = os.fdopen(handle, "w+")
 727             if self.isWindows:
 728                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                 separatorLine = separatorLine.replace("\n", "\r\n")
 730                 newdiff = newdiff.replace("\n", "\r\n")
 731             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732             tmpFile.close()
 733             mtime = os.stat(fileName).st_mtime
 734             defaultEditor = "vi"
 735             if platform.system() == "Windows":
 736                 defaultEditor = "notepad"
 737             if os.environ.has_key("P4EDITOR"):
 738                 editor = os.environ.get("P4EDITOR")
 739             else:
 740                 editor = os.environ.get("EDITOR", defaultEditor);
 741             system(editor + " " + fileName)
 742
 743             response = "y"
 744             if os.stat(fileName).st_mtime <= mtime:
 745                 response = "x"
 746                 while response != "y" and response != "n":
 747                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 748
 749             if response == "y":
 750                 tmpFile = open(fileName, "rb")
 751                 message = tmpFile.read()
 752                 tmpFile.close()
 753                 submitTemplate = message[:message.index(separatorLine)]
 754                 if self.isWindows:
 755                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 756                 p4_write_pipe("submit -i", submitTemplate)
 757             else:
 758                 for f in editedFiles:
 759                     p4_system("revert \"%s\"" % f);
 760                 for f in filesToAdd:
 761                     p4_system("revert \"%s\"" % f);
 762                     system("rm %s" %f)
 763
 764             os.remove(fileName)
 765         else:
 766             fileName = "submit.txt"
 767             file = open(fileName, "w+")
 768             file.write(self.prepareLogMessage(template, logMessage))
 769             file.close()
 770             print ("Perforce submit template written as %s. "
 771                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 772                    % (fileName, fileName))
 773
 774     def run(self, args):
 775         if len(args) == 0:
 776             self.master = currentGitBranch()
 777             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 778                 die("Detecting current git branch failed!")
 779         elif len(args) == 1:
 780             self.master = args[0]
 781         else:
 782             return False
 783
 784         allowSubmit = gitConfig("git-p4.allowSubmit")
 785         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 786             die("%s is not in git-p4.allowSubmit" % self.master)
 787
 788         [upstream, settings] = findUpstreamBranchPoint()
 789         self.depotPath = settings['depot-paths'][0]
 790         if len(self.origin) == 0:
 791             self.origin = upstream
 792
 793         if self.verbose:
 794             print "Origin branch is " + self.origin
 795
 796         if len(self.depotPath) == 0:
 797             print "Internal error: cannot locate perforce depot path from existing branches"
 798             sys.exit(128)
 799
 800         self.clientPath = p4Where(self.depotPath)
 801
 802         if len(self.clientPath) == 0:
 803             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 804             sys.exit(128)
 805
 806         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 807         self.oldWorkingDirectory = os.getcwd()
 808
 809         chdir(self.clientPath)
 810         print "Syncronizing p4 checkout..."
 811         p4_system("sync ...")
 812
 813         self.check()
 814
 815         commits = []
 816         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 817             commits.append(line.strip())
 818         commits.reverse()
 819
 820         while len(commits) > 0:
 821             commit = commits[0]
 822             commits = commits[1:]
 823             self.applyCommit(commit)
 824             if not self.interactive:
 825                 break
 826
 827         if len(commits) == 0:
 828             print "All changes applied!"
 829             chdir(self.oldWorkingDirectory)
 830
 831             sync = P4Sync()
 832             sync.run([])
 833
 834             rebase = P4Rebase()
 835             rebase.rebase()
 836
 837         return True
 838
 839 class P4Sync(Command):
 840     def __init__(self):
 841         Command.__init__(self)
 842         self.options = [
 843                 optparse.make_option("--branch", dest="branch"),
 844                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 845                 optparse.make_option("--changesfile", dest="changesFile"),
 846                 optparse.make_option("--silent", dest="silent", action="store_true"),
 847                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 848                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 849                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 850                                      help="Import into refs/heads/ , not refs/remotes"),
 851                 optparse.make_option("--max-changes", dest="maxChanges"),
 852                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 853                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 854                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 855                                      help="Only sync files that are included in the Perforce Client Spec")
 856         ]
 857         self.description = """Imports from Perforce into a git repository.\n
 858     example:
 859     //depot/my/project/ -- to import the current head
 860     //depot/my/project/@all -- to import everything
 861     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 862
 863     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 864
 865         self.usage += " //depot/path[@revRange]"
 866         self.silent = False
 867         self.createdBranches = Set()
 868         self.committedChanges = Set()
 869         self.branch = ""
 870         self.detectBranches = False
 871         self.detectLabels = False
 872         self.changesFile = ""
 873         self.syncWithOrigin = True
 874         self.verbose = False
 875         self.importIntoRemotes = True
 876         self.maxChanges = ""
 877         self.isWindows = (platform.system() == "Windows")
 878         self.keepRepoPath = False
 879         self.depotPaths = None
 880         self.p4BranchesInGit = []
 881         self.cloneExclude = []
 882         self.useClientSpec = False
 883         self.clientSpecDirs = []
 884
 885         if gitConfig("git-p4.syncFromOrigin") == "false":
 886             self.syncWithOrigin = False
 887
 888     def extractFilesFromCommit(self, commit):
 889         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 890                              for path in self.cloneExclude]
 891         files = []
 892         fnum = 0
 893         while commit.has_key("depotFile%s" % fnum):
 894             path =  commit["depotFile%s" % fnum]
 895
 896             if [p for p in self.cloneExclude
 897                 if path.startswith (p)]:
 898                 found = False
 899             else:
 900                 found = [p for p in self.depotPaths
 901                          if path.startswith (p)]
 902             if not found:
 903                 fnum = fnum + 1
 904                 continue
 905
 906             file = {}
 907             file["path"] = path
 908             file["rev"] = commit["rev%s" % fnum]
 909             file["action"] = commit["action%s" % fnum]
 910             file["type"] = commit["type%s" % fnum]
 911             files.append(file)
 912             fnum = fnum + 1
 913         return files
 914
 915     def stripRepoPath(self, path, prefixes):
 916         if self.keepRepoPath:
 917             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 918
 919         for p in prefixes:
 920             if path.startswith(p):
 921                 path = path[len(p):]
 922
 923         return path
 924
 925     def splitFilesIntoBranches(self, commit):
 926         branches = {}
 927         fnum = 0
 928         while commit.has_key("depotFile%s" % fnum):
 929             path =  commit["depotFile%s" % fnum]
 930             found = [p for p in self.depotPaths
 931                      if path.startswith (p)]
 932             if not found:
 933                 fnum = fnum + 1
 934                 continue
 935
 936             file = {}
 937             file["path"] = path
 938             file["rev"] = commit["rev%s" % fnum]
 939             file["action"] = commit["action%s" % fnum]
 940             file["type"] = commit["type%s" % fnum]
 941             fnum = fnum + 1
 942
 943             relPath = self.stripRepoPath(path, self.depotPaths)
 944
 945             for branch in self.knownBranches.keys():
 946
 947                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 948                 if relPath.startswith(branch + "/"):
 949                     if branch not in branches:
 950                         branches[branch] = []
 951                     branches[branch].append(file)
 952                     break
 953
 954         return branches
 955
 956     # output one file from the P4 stream
 957     # - helper for streamP4Files
 958
 959     def streamOneP4File(self, file, contents):
 960         if file["type"] == "apple":
 961             print "\nfile %s is a strange apple file that forks. Ignoring" % \
 962                 file['depotFile']
 963             return
 964
 965         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 966         if verbose:
 967             sys.stderr.write("%s\n" % relPath)
 968
 969         mode = "644"
 970         if isP4Exec(file["type"]):
 971             mode = "755"
 972         elif file["type"] == "symlink":
 973             mode = "120000"
 974             # p4 print on a symlink contains "target\n", so strip it off
 975             last = contents.pop()
 976             last = last[:-1]
 977             contents.append(last)
 978
 979         if self.isWindows and file["type"].endswith("text"):
 980             mangled = []
 981             for data in contents:
 982                 data = data.replace("\r\n", "\n")
 983                 mangled.append(data)
 984             contents = mangled
 985
 986         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 987             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 988         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 989             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 990
 991         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 992
 993         # total length...
 994         length = 0
 995         for d in contents:
 996             length = length + len(d)
 997
 998         self.gitStream.write("data %d\n" % length)
 999         for d in contents:
1000             self.gitStream.write(d)
1001         self.gitStream.write("\n")
1002
1003     def streamOneP4Deletion(self, file):
1004         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1005         if verbose:
1006             sys.stderr.write("delete %s\n" % relPath)
1007         self.gitStream.write("D %s\n" % relPath)
1008
1009     # handle another chunk of streaming data
1010     def streamP4FilesCb(self, marshalled):
1011
1012         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1013             # start of a new file - output the old one first
1014             self.streamOneP4File(self.stream_file, self.stream_contents)
1015             self.stream_file = {}
1016             self.stream_contents = []
1017             self.stream_have_file_info = False
1018
1019         # pick up the new file information... for the
1020         # 'data' field we need to append to our array
1021         for k in marshalled.keys():
1022             if k == 'data':
1023                 self.stream_contents.append(marshalled['data'])
1024             else:
1025                 self.stream_file[k] = marshalled[k]
1026
1027         self.stream_have_file_info = True
1028
1029     # Stream directly from "p4 files" into "git fast-import"
1030     def streamP4Files(self, files):
1031         filesForCommit = []
1032         filesToRead = []
1033         filesToDelete = []
1034
1035         for f in files:
1036             includeFile = True
1037             for val in self.clientSpecDirs:
1038                 if f['path'].startswith(val[0]):
1039                     if val[1] <= 0:
1040                         includeFile = False
1041                     break
1042
1043             if includeFile:
1044                 filesForCommit.append(f)
1045                 if f['action'] not in ('delete', 'purge'):
1046                     filesToRead.append(f)
1047                 else:
1048                     filesToDelete.append(f)
1049
1050         # deleted files...
1051         for f in filesToDelete:
1052             self.streamOneP4Deletion(f)
1053
1054         if len(filesToRead) > 0:
1055             self.stream_file = {}
1056             self.stream_contents = []
1057             self.stream_have_file_info = False
1058
1059             # curry self argument
1060             def streamP4FilesCbSelf(entry):
1061                 self.streamP4FilesCb(entry)
1062
1063             p4CmdList("-x - print",
1064                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1065                                                   for f in filesToRead]),
1066                 cb=streamP4FilesCbSelf)
1067
1068             # do the last chunk
1069             if self.stream_file.has_key('depotFile'):
1070                 self.streamOneP4File(self.stream_file, self.stream_contents)
1071
1072     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1073         epoch = details["time"]
1074         author = details["user"]
1075         self.branchPrefixes = branchPrefixes
1076
1077         if self.verbose:
1078             print "commit into %s" % branch
1079
1080         # start with reading files; if that fails, we should not
1081         # create a commit.
1082         new_files = []
1083         for f in files:
1084             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1085                 new_files.append (f)
1086             else:
1087                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1088
1089         self.gitStream.write("commit %s\n" % branch)
1090 #        gitStream.write("mark :%s\n" % details["change"])
1091         self.committedChanges.add(int(details["change"]))
1092         committer = ""
1093         if author not in self.users:
1094             self.getUserMapFromPerforceServer()
1095         if author in self.users:
1096             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1097         else:
1098             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1099
1100         self.gitStream.write("committer %s\n" % committer)
1101
1102         self.gitStream.write("data <<EOT\n")
1103         self.gitStream.write(details["desc"])
1104         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1105                              % (','.join (branchPrefixes), details["change"]))
1106         if len(details['options']) > 0:
1107             self.gitStream.write(": options = %s" % details['options'])
1108         self.gitStream.write("]\nEOT\n\n")
1109
1110         if len(parent) > 0:
1111             if self.verbose:
1112                 print "parent %s" % parent
1113             self.gitStream.write("from %s\n" % parent)
1114
1115         self.streamP4Files(new_files)
1116         self.gitStream.write("\n")
1117
1118         change = int(details["change"])
1119
1120         if self.labels.has_key(change):
1121             label = self.labels[change]
1122             labelDetails = label[0]
1123             labelRevisions = label[1]
1124             if self.verbose:
1125                 print "Change %s is labelled %s" % (change, labelDetails)
1126
1127             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1128                                                     for p in branchPrefixes]))
1129
1130             if len(files) == len(labelRevisions):
1131
1132                 cleanedFiles = {}
1133                 for info in files:
1134                     if info["action"] in ("delete", "purge"):
1135                         continue
1136                     cleanedFiles[info["depotFile"]] = info["rev"]
1137
1138                 if cleanedFiles == labelRevisions:
1139                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1140                     self.gitStream.write("from %s\n" % branch)
1141
1142                     owner = labelDetails["Owner"]
1143                     tagger = ""
1144                     if author in self.users:
1145                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1146                     else:
1147                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1148                     self.gitStream.write("tagger %s\n" % tagger)
1149                     self.gitStream.write("data <<EOT\n")
1150                     self.gitStream.write(labelDetails["Description"])
1151                     self.gitStream.write("EOT\n\n")
1152
1153                 else:
1154                     if not self.silent:
1155                         print ("Tag %s does not match with change %s: files do not match."
1156                                % (labelDetails["label"], change))
1157
1158             else:
1159                 if not self.silent:
1160                     print ("Tag %s does not match with change %s: file count is different."
1161                            % (labelDetails["label"], change))
1162
1163     def getUserCacheFilename(self):
1164         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1165         return home + "/.gitp4-usercache.txt"
1166
1167     def getUserMapFromPerforceServer(self):
1168         if self.userMapFromPerforceServer:
1169             return
1170         self.users = {}
1171
1172         for output in p4CmdList("users"):
1173             if not output.has_key("User"):
1174                 continue
1175             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1176
1177
1178         s = ''
1179         for (key, val) in self.users.items():
1180             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1181
1182         open(self.getUserCacheFilename(), "wb").write(s)
1183         self.userMapFromPerforceServer = True
1184
1185     def loadUserMapFromCache(self):
1186         self.users = {}
1187         self.userMapFromPerforceServer = False
1188         try:
1189             cache = open(self.getUserCacheFilename(), "rb")
1190             lines = cache.readlines()
1191             cache.close()
1192             for line in lines:
1193                 entry = line.strip().split("\t")
1194                 self.users[entry[0]] = entry[1]
1195         except IOError:
1196             self.getUserMapFromPerforceServer()
1197
1198     def getLabels(self):
1199         self.labels = {}
1200
1201         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1202         if len(l) > 0 and not self.silent:
1203             print "Finding files belonging to labels in %s" % `self.depotPaths`
1204
1205         for output in l:
1206             label = output["label"]
1207             revisions = {}
1208             newestChange = 0
1209             if self.verbose:
1210                 print "Querying files for label %s" % label
1211             for file in p4CmdList("files "
1212                                   +  ' '.join (["%s...@%s" % (p, label)
1213                                                 for p in self.depotPaths])):
1214                 revisions[file["depotFile"]] = file["rev"]
1215                 change = int(file["change"])
1216                 if change > newestChange:
1217                     newestChange = change
1218
1219             self.labels[newestChange] = [output, revisions]
1220
1221         if self.verbose:
1222             print "Label changes: %s" % self.labels.keys()
1223
1224     def guessProjectName(self):
1225         for p in self.depotPaths:
1226             if p.endswith("/"):
1227                 p = p[:-1]
1228             p = p[p.strip().rfind("/") + 1:]
1229             if not p.endswith("/"):
1230                p += "/"
1231             return p
1232
1233     def getBranchMapping(self):
1234         lostAndFoundBranches = set()
1235
1236         for info in p4CmdList("branches"):
1237             details = p4Cmd("branch -o %s" % info["branch"])
1238             viewIdx = 0
1239             while details.has_key("View%s" % viewIdx):
1240                 paths = details["View%s" % viewIdx].split(" ")
1241                 viewIdx = viewIdx + 1
1242                 # require standard //depot/foo/... //depot/bar/... mapping
1243                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1244                     continue
1245                 source = paths[0]
1246                 destination = paths[1]
1247                 ## HACK
1248                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1249                     source = source[len(self.depotPaths[0]):-4]
1250                     destination = destination[len(self.depotPaths[0]):-4]
1251
1252                     if destination in self.knownBranches:
1253                         if not self.silent:
1254                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1255                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1256                         continue
1257
1258                     self.knownBranches[destination] = source
1259
1260                     lostAndFoundBranches.discard(destination)
1261
1262                     if source not in self.knownBranches:
1263                         lostAndFoundBranches.add(source)
1264
1265
1266         for branch in lostAndFoundBranches:
1267             self.knownBranches[branch] = branch
1268
1269     def getBranchMappingFromGitBranches(self):
1270         branches = p4BranchesInGit(self.importIntoRemotes)
1271         for branch in branches.keys():
1272             if branch == "master":
1273                 branch = "main"
1274             else:
1275                 branch = branch[len(self.projectName):]
1276             self.knownBranches[branch] = branch
1277
1278     def listExistingP4GitBranches(self):
1279         # branches holds mapping from name to commit
1280         branches = p4BranchesInGit(self.importIntoRemotes)
1281         self.p4BranchesInGit = branches.keys()
1282         for branch in branches.keys():
1283             self.initialParents[self.refPrefix + branch] = branches[branch]
1284
1285     def updateOptionDict(self, d):
1286         option_keys = {}
1287         if self.keepRepoPath:
1288             option_keys['keepRepoPath'] = 1
1289
1290         d["options"] = ' '.join(sorted(option_keys.keys()))
1291
1292     def readOptions(self, d):
1293         self.keepRepoPath = (d.has_key('options')
1294                              and ('keepRepoPath' in d['options']))
1295
1296     def gitRefForBranch(self, branch):
1297         if branch == "main":
1298             return self.refPrefix + "master"
1299
1300         if len(branch) <= 0:
1301             return branch
1302
1303         return self.refPrefix + self.projectName + branch
1304
1305     def gitCommitByP4Change(self, ref, change):
1306         if self.verbose:
1307             print "looking in ref " + ref + " for change %s using bisect..." % change
1308
1309         earliestCommit = ""
1310         latestCommit = parseRevision(ref)
1311
1312         while True:
1313             if self.verbose:
1314                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1315             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1316             if len(next) == 0:
1317                 if self.verbose:
1318                     print "argh"
1319                 return ""
1320             log = extractLogMessageFromGitCommit(next)
1321             settings = extractSettingsGitLog(log)
1322             currentChange = int(settings['change'])
1323             if self.verbose:
1324                 print "current change %s" % currentChange
1325
1326             if currentChange == change:
1327                 if self.verbose:
1328                     print "found %s" % next
1329                 return next
1330
1331             if currentChange < change:
1332                 earliestCommit = "^%s" % next
1333             else:
1334                 latestCommit = "%s" % next
1335
1336         return ""
1337
1338     def importNewBranch(self, branch, maxChange):
1339         # make fast-import flush all changes to disk and update the refs using the checkpoint
1340         # command so that we can try to find the branch parent in the git history
1341         self.gitStream.write("checkpoint\n\n");
1342         self.gitStream.flush();
1343         branchPrefix = self.depotPaths[0] + branch + "/"
1344         range = "@1,%s" % maxChange
1345         #print "prefix" + branchPrefix
1346         changes = p4ChangesForPaths([branchPrefix], range)
1347         if len(changes) <= 0:
1348             return False
1349         firstChange = changes[0]
1350         #print "first change in branch: %s" % firstChange
1351         sourceBranch = self.knownBranches[branch]
1352         sourceDepotPath = self.depotPaths[0] + sourceBranch
1353         sourceRef = self.gitRefForBranch(sourceBranch)
1354         #print "source " + sourceBranch
1355
1356         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1357         #print "branch parent: %s" % branchParentChange
1358         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1359         if len(gitParent) > 0:
1360             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1361             #print "parent git commit: %s" % gitParent
1362
1363         self.importChanges(changes)
1364         return True
1365
1366     def importChanges(self, changes):
1367         cnt = 1
1368         for change in changes:
1369             description = p4Cmd("describe %s" % change)
1370             self.updateOptionDict(description)
1371
1372             if not self.silent:
1373                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1374                 sys.stdout.flush()
1375             cnt = cnt + 1
1376
1377             try:
1378                 if self.detectBranches:
1379                     branches = self.splitFilesIntoBranches(description)
1380                     for branch in branches.keys():
1381                         ## HACK  --hwn
1382                         branchPrefix = self.depotPaths[0] + branch + "/"
1383
1384                         parent = ""
1385
1386                         filesForCommit = branches[branch]
1387
1388                         if self.verbose:
1389                             print "branch is %s" % branch
1390
1391                         self.updatedBranches.add(branch)
1392
1393                         if branch not in self.createdBranches:
1394                             self.createdBranches.add(branch)
1395                             parent = self.knownBranches[branch]
1396                             if parent == branch:
1397                                 parent = ""
1398                             else:
1399                                 fullBranch = self.projectName + branch
1400                                 if fullBranch not in self.p4BranchesInGit:
1401                                     if not self.silent:
1402                                         print("\n    Importing new branch %s" % fullBranch);
1403                                     if self.importNewBranch(branch, change - 1):
1404                                         parent = ""
1405                                         self.p4BranchesInGit.append(fullBranch)
1406                                     if not self.silent:
1407                                         print("\n    Resuming with change %s" % change);
1408
1409                                 if self.verbose:
1410                                     print "parent determined through known branches: %s" % parent
1411
1412                         branch = self.gitRefForBranch(branch)
1413                         parent = self.gitRefForBranch(parent)
1414
1415                         if self.verbose:
1416                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1417
1418                         if len(parent) == 0 and branch in self.initialParents:
1419                             parent = self.initialParents[branch]
1420                             del self.initialParents[branch]
1421
1422                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1423                 else:
1424                     files = self.extractFilesFromCommit(description)
1425                     self.commit(description, files, self.branch, self.depotPaths,
1426                                 self.initialParent)
1427                     self.initialParent = ""
1428             except IOError:
1429                 print self.gitError.read()
1430                 sys.exit(1)
1431
1432     def importHeadRevision(self, revision):
1433         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1434
1435         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1436         details["desc"] = ("Initial import of %s from the state at revision %s"
1437                            % (' '.join(self.depotPaths), revision))
1438         details["change"] = revision
1439         newestRevision = 0
1440
1441         fileCnt = 0
1442         for info in p4CmdList("files "
1443                               +  ' '.join(["%s...%s"
1444                                            % (p, revision)
1445                                            for p in self.depotPaths])):
1446
1447             if info['code'] == 'error':
1448                 sys.stderr.write("p4 returned an error: %s\n"
1449                                  % info['data'])
1450                 sys.exit(1)
1451
1452
1453             change = int(info["change"])
1454             if change > newestRevision:
1455                 newestRevision = change
1456
1457             if info["action"] in ("delete", "purge"):
1458                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1459                 #fileCnt = fileCnt + 1
1460                 continue
1461
1462             for prop in ["depotFile", "rev", "action", "type" ]:
1463                 details["%s%s" % (prop, fileCnt)] = info[prop]
1464
1465             fileCnt = fileCnt + 1
1466
1467         details["change"] = newestRevision
1468         self.updateOptionDict(details)
1469         try:
1470             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1471         except IOError:
1472             print "IO error with git fast-import. Is your git version recent enough?"
1473             print self.gitError.read()
1474
1475
1476     def getClientSpec(self):
1477         specList = p4CmdList( "client -o" )
1478         temp = {}
1479         for entry in specList:
1480             for k,v in entry.iteritems():
1481                 if k.startswith("View"):
1482                     if v.startswith('"'):
1483                         start = 1
1484                     else:
1485                         start = 0
1486                     index = v.find("...")
1487                     v = v[start:index]
1488                     if v.startswith("-"):
1489                         v = v[1:]
1490                         temp[v] = -len(v)
1491                     else:
1492                         temp[v] = len(v)
1493         self.clientSpecDirs = temp.items()
1494         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1495
1496     def run(self, args):
1497         self.depotPaths = []
1498         self.changeRange = ""
1499         self.initialParent = ""
1500         self.previousDepotPaths = []
1501
1502         # map from branch depot path to parent branch
1503         self.knownBranches = {}
1504         self.initialParents = {}
1505         self.hasOrigin = originP4BranchesExist()
1506         if not self.syncWithOrigin:
1507             self.hasOrigin = False
1508
1509         if self.importIntoRemotes:
1510             self.refPrefix = "refs/remotes/p4/"
1511         else:
1512             self.refPrefix = "refs/heads/p4/"
1513
1514         if self.syncWithOrigin and self.hasOrigin:
1515             if not self.silent:
1516                 print "Syncing with origin first by calling git fetch origin"
1517             system("git fetch origin")
1518
1519         if len(self.branch) == 0:
1520             self.branch = self.refPrefix + "master"
1521             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1522                 system("git update-ref %s refs/heads/p4" % self.branch)
1523                 system("git branch -D p4");
1524             # create it /after/ importing, when master exists
1525             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1526                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1527
1528         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1529             self.getClientSpec()
1530
1531         # TODO: should always look at previous commits,
1532         # merge with previous imports, if possible.
1533         if args == []:
1534             if self.hasOrigin:
1535                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1536             self.listExistingP4GitBranches()
1537
1538             if len(self.p4BranchesInGit) > 1:
1539                 if not self.silent:
1540                     print "Importing from/into multiple branches"
1541                 self.detectBranches = True
1542
1543             if self.verbose:
1544                 print "branches: %s" % self.p4BranchesInGit
1545
1546             p4Change = 0
1547             for branch in self.p4BranchesInGit:
1548                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1549
1550                 settings = extractSettingsGitLog(logMsg)
1551
1552                 self.readOptions(settings)
1553                 if (settings.has_key('depot-paths')
1554                     and settings.has_key ('change')):
1555                     change = int(settings['change']) + 1
1556                     p4Change = max(p4Change, change)
1557
1558                     depotPaths = sorted(settings['depot-paths'])
1559                     if self.previousDepotPaths == []:
1560                         self.previousDepotPaths = depotPaths
1561                     else:
1562                         paths = []
1563                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1564                             for i in range(0, min(len(cur), len(prev))):
1565                                 if cur[i] <> prev[i]:
1566                                     i = i - 1
1567                                     break
1568
1569                             paths.append (cur[:i + 1])
1570
1571                         self.previousDepotPaths = paths
1572
1573             if p4Change > 0:
1574                 self.depotPaths = sorted(self.previousDepotPaths)
1575                 self.changeRange = "@%s,#head" % p4Change
1576                 if not self.detectBranches:
1577                     self.initialParent = parseRevision(self.branch)
1578                 if not self.silent and not self.detectBranches:
1579                     print "Performing incremental import into %s git branch" % self.branch
1580
1581         if not self.branch.startswith("refs/"):
1582             self.branch = "refs/heads/" + self.branch
1583
1584         if len(args) == 0 and self.depotPaths:
1585             if not self.silent:
1586                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1587         else:
1588             if self.depotPaths and self.depotPaths != args:
1589                 print ("previous import used depot path %s and now %s was specified. "
1590                        "This doesn't work!" % (' '.join (self.depotPaths),
1591                                                ' '.join (args)))
1592                 sys.exit(1)
1593
1594             self.depotPaths = sorted(args)
1595
1596         revision = ""
1597         self.users = {}
1598
1599         newPaths = []
1600         for p in self.depotPaths:
1601             if p.find("@") != -1:
1602                 atIdx = p.index("@")
1603                 self.changeRange = p[atIdx:]
1604                 if self.changeRange == "@all":
1605                     self.changeRange = ""
1606                 elif ',' not in self.changeRange:
1607                     revision = self.changeRange
1608                     self.changeRange = ""
1609                 p = p[:atIdx]
1610             elif p.find("#") != -1:
1611                 hashIdx = p.index("#")
1612                 revision = p[hashIdx:]
1613                 p = p[:hashIdx]
1614             elif self.previousDepotPaths == []:
1615                 revision = "#head"
1616
1617             p = re.sub ("\.\.\.$", "", p)
1618             if not p.endswith("/"):
1619                 p += "/"
1620
1621             newPaths.append(p)
1622
1623         self.depotPaths = newPaths
1624
1625
1626         self.loadUserMapFromCache()
1627         self.labels = {}
1628         if self.detectLabels:
1629             self.getLabels();
1630
1631         if self.detectBranches:
1632             ## FIXME - what's a P4 projectName ?
1633             self.projectName = self.guessProjectName()
1634
1635             if self.hasOrigin:
1636                 self.getBranchMappingFromGitBranches()
1637             else:
1638                 self.getBranchMapping()
1639             if self.verbose:
1640                 print "p4-git branches: %s" % self.p4BranchesInGit
1641                 print "initial parents: %s" % self.initialParents
1642             for b in self.p4BranchesInGit:
1643                 if b != "master":
1644
1645                     ## FIXME
1646                     b = b[len(self.projectName):]
1647                 self.createdBranches.add(b)
1648
1649         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1650
1651         importProcess = subprocess.Popen(["git", "fast-import"],
1652                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1653                                          stderr=subprocess.PIPE);
1654         self.gitOutput = importProcess.stdout
1655         self.gitStream = importProcess.stdin
1656         self.gitError = importProcess.stderr
1657
1658         if revision:
1659             self.importHeadRevision(revision)
1660         else:
1661             changes = []
1662
1663             if len(self.changesFile) > 0:
1664                 output = open(self.changesFile).readlines()
1665                 changeSet = Set()
1666                 for line in output:
1667                     changeSet.add(int(line))
1668
1669                 for change in changeSet:
1670                     changes.append(change)
1671
1672                 changes.sort()
1673             else:
1674                 if self.verbose:
1675                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1676                                                               self.changeRange)
1677                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1678
1679                 if len(self.maxChanges) > 0:
1680                     changes = changes[:min(int(self.maxChanges), len(changes))]
1681
1682             if len(changes) == 0:
1683                 if not self.silent:
1684                     print "No changes to import!"
1685                 return True
1686
1687             if not self.silent and not self.detectBranches:
1688                 print "Import destination: %s" % self.branch
1689
1690             self.updatedBranches = set()
1691
1692             self.importChanges(changes)
1693
1694             if not self.silent:
1695                 print ""
1696                 if len(self.updatedBranches) > 0:
1697                     sys.stdout.write("Updated branches: ")
1698                     for b in self.updatedBranches:
1699                         sys.stdout.write("%s " % b)
1700                     sys.stdout.write("\n")
1701
1702         self.gitStream.close()
1703         if importProcess.wait() != 0:
1704             die("fast-import failed: %s" % self.gitError.read())
1705         self.gitOutput.close()
1706         self.gitError.close()
1707
1708         return True
1709
1710 class P4Rebase(Command):
1711     def __init__(self):
1712         Command.__init__(self)
1713         self.options = [ ]
1714         self.description = ("Fetches the latest revision from perforce and "
1715                             + "rebases the current work (branch) against it")
1716         self.verbose = False
1717
1718     def run(self, args):
1719         sync = P4Sync()
1720         sync.run([])
1721
1722         return self.rebase()
1723
1724     def rebase(self):
1725         if os.system("git update-index --refresh") != 0:
1726             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1727         if len(read_pipe("git diff-index HEAD --")) > 0:
1728             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1729
1730         [upstream, settings] = findUpstreamBranchPoint()
1731         if len(upstream) == 0:
1732             die("Cannot find upstream branchpoint for rebase")
1733
1734         # the branchpoint may be p4/foo~3, so strip off the parent
1735         upstream = re.sub("~[0-9]+$", "", upstream)
1736
1737         print "Rebasing the current branch onto %s" % upstream
1738         oldHead = read_pipe("git rev-parse HEAD").strip()
1739         system("git rebase %s" % upstream)
1740         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1741         return True
1742
1743 class P4Clone(P4Sync):
1744     def __init__(self):
1745         P4Sync.__init__(self)
1746         self.description = "Creates a new git repository and imports from Perforce into it"
1747         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1748         self.options += [
1749             optparse.make_option("--destination", dest="cloneDestination",
1750                                  action='store', default=None,
1751                                  help="where to leave result of the clone"),
1752             optparse.make_option("-/", dest="cloneExclude",
1753                                  action="append", type="string",
1754                                  help="exclude depot path")
1755         ]
1756         self.cloneDestination = None
1757         self.needsGit = False
1758
1759     # This is required for the "append" cloneExclude action
1760     def ensure_value(self, attr, value):
1761         if not hasattr(self, attr) or getattr(self, attr) is None:
1762             setattr(self, attr, value)
1763         return getattr(self, attr)
1764
1765     def defaultDestination(self, args):
1766         ## TODO: use common prefix of args?
1767         depotPath = args[0]
1768         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1769         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1770         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1771         depotDir = re.sub(r"/$", "", depotDir)
1772         return os.path.split(depotDir)[1]
1773
1774     def run(self, args):
1775         if len(args) < 1:
1776             return False
1777
1778         if self.keepRepoPath and not self.cloneDestination:
1779             sys.stderr.write("Must specify destination for --keep-path\n")
1780             sys.exit(1)
1781
1782         depotPaths = args
1783
1784         if not self.cloneDestination and len(depotPaths) > 1:
1785             self.cloneDestination = depotPaths[-1]
1786             depotPaths = depotPaths[:-1]
1787
1788         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1789         for p in depotPaths:
1790             if not p.startswith("//"):
1791                 return False
1792
1793         if not self.cloneDestination:
1794             self.cloneDestination = self.defaultDestination(args)
1795
1796         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1797         if not os.path.exists(self.cloneDestination):
1798             os.makedirs(self.cloneDestination)
1799         chdir(self.cloneDestination)
1800         system("git init")
1801         self.gitdir = os.getcwd() + "/.git"
1802         if not P4Sync.run(self, depotPaths):
1803             return False
1804         if self.branch != "master":
1805             if self.importIntoRemotes:
1806                 masterbranch = "refs/remotes/p4/master"
1807             else:
1808                 masterbranch = "refs/heads/p4/master"
1809             if gitBranchExists(masterbranch):
1810                 system("git branch master %s" % masterbranch)
1811                 system("git checkout -f")
1812             else:
1813                 print "Could not detect main branch. No checkout/master branch created."
1814
1815         return True
1816
1817 class P4Branches(Command):
1818     def __init__(self):
1819         Command.__init__(self)
1820         self.options = [ ]
1821         self.description = ("Shows the git branches that hold imports and their "
1822                             + "corresponding perforce depot paths")
1823         self.verbose = False
1824
1825     def run(self, args):
1826         if originP4BranchesExist():
1827             createOrUpdateBranchesFromOrigin()
1828
1829         cmdline = "git rev-parse --symbolic "
1830         cmdline += " --remotes"
1831
1832         for line in read_pipe_lines(cmdline):
1833             line = line.strip()
1834
1835             if not line.startswith('p4/') or line == "p4/HEAD":
1836                 continue
1837             branch = line
1838
1839             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1840             settings = extractSettingsGitLog(log)
1841
1842             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1843         return True
1844
1845 class HelpFormatter(optparse.IndentedHelpFormatter):
1846     def __init__(self):
1847         optparse.IndentedHelpFormatter.__init__(self)
1848
1849     def format_description(self, description):
1850         if description:
1851             return description + "\n"
1852         else:
1853             return ""
1854
1855 def printUsage(commands):
1856     print "usage: %s <command> [options]" % sys.argv[0]
1857     print ""
1858     print "valid commands: %s" % ", ".join(commands)
1859     print ""
1860     print "Try %s <command> --help for command specific help." % sys.argv[0]
1861     print ""
1862
1863 commands = {
1864     "debug" : P4Debug,
1865     "submit" : P4Submit,
1866     "commit" : P4Submit,
1867     "sync" : P4Sync,
1868     "rebase" : P4Rebase,
1869     "clone" : P4Clone,
1870     "rollback" : P4RollBack,
1871     "branches" : P4Branches
1872 }
1873
1874
1875 def main():
1876     if len(sys.argv[1:]) == 0:
1877         printUsage(commands.keys())
1878         sys.exit(2)
1879
1880     cmd = ""
1881     cmdName = sys.argv[1]
1882     try:
1883         klass = commands[cmdName]
1884         cmd = klass()
1885     except KeyError:
1886         print "unknown command %s" % cmdName
1887         print ""
1888         printUsage(commands.keys())
1889         sys.exit(2)
1890
1891     options = cmd.options
1892     cmd.gitdir = os.environ.get("GIT_DIR", None)
1893
1894     args = sys.argv[2:]
1895
1896     if len(options) > 0:
1897         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1898
1899         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1900                                        options,
1901                                        description = cmd.description,
1902                                        formatter = HelpFormatter())
1903
1904         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1905     global verbose
1906     verbose = cmd.verbose
1907     if cmd.needsGit:
1908         if cmd.gitdir == None:
1909             cmd.gitdir = os.path.abspath(".git")
1910             if not isValidGitDir(cmd.gitdir):
1911                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1912                 if os.path.exists(cmd.gitdir):
1913                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1914                     if len(cdup) > 0:
1915                         chdir(cdup);
1916
1917         if not isValidGitDir(cmd.gitdir):
1918             if isValidGitDir(cmd.gitdir + "/.git"):
1919                 cmd.gitdir += "/.git"
1920             else:
1921                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1922
1923         os.environ["GIT_DIR"] = cmd.gitdir
1924
1925     if not cmd.run(args):
1926         parser.print_help()
1927
1928
1929 if __name__ == '__main__':
1930     main()