contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def die(msg):
  55     if verbose:
  56         raise Exception(msg)
  57     else:
  58         sys.stderr.write(msg + "\n")
  59         sys.exit(1)
  60
  61 def write_pipe(c, str):
  62     if verbose:
  63         sys.stderr.write('Writing pipe: %s\n' % c)
  64
  65     pipe = os.popen(c, 'w')
  66     val = pipe.write(str)
  67     if pipe.close():
  68         die('Command failed: %s' % c)
  69
  70     return val
  71
  72 def read_pipe(c, ignore_error=False):
  73     if verbose:
  74         sys.stderr.write('Reading pipe: %s\n' % c)
  75
  76     pipe = os.popen(c, 'rb')
  77     val = pipe.read()
  78     if pipe.close() and not ignore_error:
  79         die('Command failed: %s' % c)
  80
  81     return val
  82
  83
  84 def read_pipe_lines(c):
  85     if verbose:
  86         sys.stderr.write('Reading pipe: %s\n' % c)
  87     ## todo: check return status
  88     pipe = os.popen(c, 'rb')
  89     val = pipe.readlines()
  90     if pipe.close():
  91         die('Command failed: %s' % c)
  92
  93     return val
  94
  95 def p4_read_pipe_lines(c):
  96     """Specifically invoke p4 on the command supplied. """
  97     real_cmd = p4_build_cmd(c)
  98     return read_pipe_lines(real_cmd)
  99
 100 def system(cmd):
 101     if verbose:
 102         sys.stderr.write("executing %s\n" % cmd)
 103     if os.system(cmd) != 0:
 104         die("command failed: %s" % cmd)
 105
 106 def p4_system(cmd):
 107     """Specifically invoke p4 as the system command. """
 108     real_cmd = p4_build_cmd(cmd)
 109     return system(real_cmd)
 110
 111 def isP4Exec(kind):
 112     """Determine if a Perforce 'kind' should have execute permission
 113
 114     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 115     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 116     a plus sign, it is also executable"""
 117     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 118
 119 def setP4ExecBit(file, mode):
 120     # Reopens an already open file and changes the execute bit to match
 121     # the execute bit setting in the passed in mode.
 122
 123     p4Type = "+x"
 124
 125     if not isModeExec(mode):
 126         p4Type = getP4OpenedType(file)
 127         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 128         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 129         if p4Type[-1] == "+":
 130             p4Type = p4Type[0:-1]
 131
 132     p4_system("reopen -t %s %s" % (p4Type, file))
 133
 134 def getP4OpenedType(file):
 135     # Returns the perforce file type for the given file.
 136
 137     result = read_pipe("p4 opened %s" % file)
 138     match = re.match(".*\((.+)\)\r?$", result)
 139     if match:
 140         return match.group(1)
 141     else:
 142         die("Could not determine file type for %s (result: '%s')" % (file, result))
 143
 144 def diffTreePattern():
 145     # This is a simple generator for the diff tree regex pattern. This could be
 146     # a class variable if this and parseDiffTreeEntry were a part of a class.
 147     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 148     while True:
 149         yield pattern
 150
 151 def parseDiffTreeEntry(entry):
 152     """Parses a single diff tree entry into its component elements.
 153
 154     See git-diff-tree(1) manpage for details about the format of the diff
 155     output. This method returns a dictionary with the following elements:
 156
 157     src_mode - The mode of the source file
 158     dst_mode - The mode of the destination file
 159     src_sha1 - The sha1 for the source file
 160     dst_sha1 - The sha1 fr the destination file
 161     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 162     status_score - The score for the status (applicable for 'C' and 'R'
 163                    statuses). This is None if there is no score.
 164     src - The path for the source file.
 165     dst - The path for the destination file. This is only present for
 166           copy or renames. If it is not present, this is None.
 167
 168     If the pattern is not matched, None is returned."""
 169
 170     match = diffTreePattern().next().match(entry)
 171     if match:
 172         return {
 173             'src_mode': match.group(1),
 174             'dst_mode': match.group(2),
 175             'src_sha1': match.group(3),
 176             'dst_sha1': match.group(4),
 177             'status': match.group(5),
 178             'status_score': match.group(6),
 179             'src': match.group(7),
 180             'dst': match.group(10)
 181         }
 182     return None
 183
 184 def isModeExec(mode):
 185     # Returns True if the given git mode represents an executable file,
 186     # otherwise False.
 187     return mode[-3:] == "755"
 188
 189 def isModeExecChanged(src_mode, dst_mode):
 190     return isModeExec(src_mode) != isModeExec(dst_mode)
 191
 192 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 193     cmd = p4_build_cmd("-G %s" % (cmd))
 194     if verbose:
 195         sys.stderr.write("Opening pipe: %s\n" % cmd)
 196
 197     # Use a temporary file to avoid deadlocks without
 198     # subprocess.communicate(), which would put another copy
 199     # of stdout into memory.
 200     stdin_file = None
 201     if stdin is not None:
 202         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 203         stdin_file.write(stdin)
 204         stdin_file.flush()
 205         stdin_file.seek(0)
 206
 207     p4 = subprocess.Popen(cmd, shell=True,
 208                           stdin=stdin_file,
 209                           stdout=subprocess.PIPE)
 210
 211     result = []
 212     try:
 213         while True:
 214             entry = marshal.load(p4.stdout)
 215             result.append(entry)
 216     except EOFError:
 217         pass
 218     exitCode = p4.wait()
 219     if exitCode != 0:
 220         entry = {}
 221         entry["p4ExitCode"] = exitCode
 222         result.append(entry)
 223
 224     return result
 225
 226 def p4Cmd(cmd):
 227     list = p4CmdList(cmd)
 228     result = {}
 229     for entry in list:
 230         result.update(entry)
 231     return result;
 232
 233 def p4Where(depotPath):
 234     if not depotPath.endswith("/"):
 235         depotPath += "/"
 236     output = p4Cmd("where %s..." % depotPath)
 237     if output["code"] == "error":
 238         return ""
 239     clientPath = ""
 240     if "path" in output:
 241         clientPath = output.get("path")
 242     elif "data" in output:
 243         data = output.get("data")
 244         lastSpace = data.rfind(" ")
 245         clientPath = data[lastSpace + 1:]
 246
 247     if clientPath.endswith("..."):
 248         clientPath = clientPath[:-3]
 249     return clientPath
 250
 251 def currentGitBranch():
 252     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 253
 254 def isValidGitDir(path):
 255     if (os.path.exists(path + "/HEAD")
 256         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 257         return True;
 258     return False
 259
 260 def parseRevision(ref):
 261     return read_pipe("git rev-parse %s" % ref).strip()
 262
 263 def extractLogMessageFromGitCommit(commit):
 264     logMessage = ""
 265
 266     ## fixme: title is first line of commit, not 1st paragraph.
 267     foundTitle = False
 268     for log in read_pipe_lines("git cat-file commit %s" % commit):
 269        if not foundTitle:
 270            if len(log) == 1:
 271                foundTitle = True
 272            continue
 273
 274        logMessage += log
 275     return logMessage
 276
 277 def extractSettingsGitLog(log):
 278     values = {}
 279     for line in log.split("\n"):
 280         line = line.strip()
 281         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 282         if not m:
 283             continue
 284
 285         assignments = m.group(1).split (':')
 286         for a in assignments:
 287             vals = a.split ('=')
 288             key = vals[0].strip()
 289             val = ('='.join (vals[1:])).strip()
 290             if val.endswith ('\"') and val.startswith('"'):
 291                 val = val[1:-1]
 292
 293             values[key] = val
 294
 295     paths = values.get("depot-paths")
 296     if not paths:
 297         paths = values.get("depot-path")
 298     if paths:
 299         values['depot-paths'] = paths.split(',')
 300     return values
 301
 302 def gitBranchExists(branch):
 303     proc = subprocess.Popen(["git", "rev-parse", branch],
 304                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 305     return proc.wait() == 0;
 306
 307 def gitConfig(key):
 308     return read_pipe("git config %s" % key, ignore_error=True).strip()
 309
 310 def p4BranchesInGit(branchesAreInRemotes = True):
 311     branches = {}
 312
 313     cmdline = "git rev-parse --symbolic "
 314     if branchesAreInRemotes:
 315         cmdline += " --remotes"
 316     else:
 317         cmdline += " --branches"
 318
 319     for line in read_pipe_lines(cmdline):
 320         line = line.strip()
 321
 322         ## only import to p4/
 323         if not line.startswith('p4/') or line == "p4/HEAD":
 324             continue
 325         branch = line
 326
 327         # strip off p4
 328         branch = re.sub ("^p4/", "", line)
 329
 330         branches[branch] = parseRevision(line)
 331     return branches
 332
 333 def findUpstreamBranchPoint(head = "HEAD"):
 334     branches = p4BranchesInGit()
 335     # map from depot-path to branch name
 336     branchByDepotPath = {}
 337     for branch in branches.keys():
 338         tip = branches[branch]
 339         log = extractLogMessageFromGitCommit(tip)
 340         settings = extractSettingsGitLog(log)
 341         if settings.has_key("depot-paths"):
 342             paths = ",".join(settings["depot-paths"])
 343             branchByDepotPath[paths] = "remotes/p4/" + branch
 344
 345     settings = None
 346     parent = 0
 347     while parent < 65535:
 348         commit = head + "~%s" % parent
 349         log = extractLogMessageFromGitCommit(commit)
 350         settings = extractSettingsGitLog(log)
 351         if settings.has_key("depot-paths"):
 352             paths = ",".join(settings["depot-paths"])
 353             if branchByDepotPath.has_key(paths):
 354                 return [branchByDepotPath[paths], settings]
 355
 356         parent = parent + 1
 357
 358     return ["", settings]
 359
 360 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 361     if not silent:
 362         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 363                % localRefPrefix)
 364
 365     originPrefix = "origin/p4/"
 366
 367     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 368         line = line.strip()
 369         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 370             continue
 371
 372         headName = line[len(originPrefix):]
 373         remoteHead = localRefPrefix + headName
 374         originHead = line
 375
 376         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 377         if (not original.has_key('depot-paths')
 378             or not original.has_key('change')):
 379             continue
 380
 381         update = False
 382         if not gitBranchExists(remoteHead):
 383             if verbose:
 384                 print "creating %s" % remoteHead
 385             update = True
 386         else:
 387             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 388             if settings.has_key('change') > 0:
 389                 if settings['depot-paths'] == original['depot-paths']:
 390                     originP4Change = int(original['change'])
 391                     p4Change = int(settings['change'])
 392                     if originP4Change > p4Change:
 393                         print ("%s (%s) is newer than %s (%s). "
 394                                "Updating p4 branch from origin."
 395                                % (originHead, originP4Change,
 396                                   remoteHead, p4Change))
 397                         update = True
 398                 else:
 399                     print ("Ignoring: %s was imported from %s while "
 400                            "%s was imported from %s"
 401                            % (originHead, ','.join(original['depot-paths']),
 402                               remoteHead, ','.join(settings['depot-paths'])))
 403
 404         if update:
 405             system("git update-ref %s %s" % (remoteHead, originHead))
 406
 407 def originP4BranchesExist():
 408         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 409
 410 def p4ChangesForPaths(depotPaths, changeRange):
 411     assert depotPaths
 412     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 413                                                         for p in depotPaths]))
 414
 415     changes = []
 416     for line in output:
 417         changeNum = line.split(" ")[1]
 418         changes.append(int(changeNum))
 419
 420     changes.sort()
 421     return changes
 422
 423 class Command:
 424     def __init__(self):
 425         self.usage = "usage: %prog [options]"
 426         self.needsGit = True
 427
 428 class P4Debug(Command):
 429     def __init__(self):
 430         Command.__init__(self)
 431         self.options = [
 432             optparse.make_option("--verbose", dest="verbose", action="store_true",
 433                                  default=False),
 434             ]
 435         self.description = "A tool to debug the output of p4 -G."
 436         self.needsGit = False
 437         self.verbose = False
 438
 439     def run(self, args):
 440         j = 0
 441         for output in p4CmdList(" ".join(args)):
 442             print 'Element: %d' % j
 443             j += 1
 444             print output
 445         return True
 446
 447 class P4RollBack(Command):
 448     def __init__(self):
 449         Command.__init__(self)
 450         self.options = [
 451             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 452             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 453         ]
 454         self.description = "A tool to debug the multi-branch import. Don't use :)"
 455         self.verbose = False
 456         self.rollbackLocalBranches = False
 457
 458     def run(self, args):
 459         if len(args) != 1:
 460             return False
 461         maxChange = int(args[0])
 462
 463         if "p4ExitCode" in p4Cmd("changes -m 1"):
 464             die("Problems executing p4");
 465
 466         if self.rollbackLocalBranches:
 467             refPrefix = "refs/heads/"
 468             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 469         else:
 470             refPrefix = "refs/remotes/"
 471             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 472
 473         for line in lines:
 474             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 475                 line = line.strip()
 476                 ref = refPrefix + line
 477                 log = extractLogMessageFromGitCommit(ref)
 478                 settings = extractSettingsGitLog(log)
 479
 480                 depotPaths = settings['depot-paths']
 481                 change = settings['change']
 482
 483                 changed = False
 484
 485                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 486                                                            for p in depotPaths]))) == 0:
 487                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 488                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 489                     continue
 490
 491                 while change and int(change) > maxChange:
 492                     changed = True
 493                     if self.verbose:
 494                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 495                     system("git update-ref %s \"%s^\"" % (ref, ref))
 496                     log = extractLogMessageFromGitCommit(ref)
 497                     settings =  extractSettingsGitLog(log)
 498
 499
 500                     depotPaths = settings['depot-paths']
 501                     change = settings['change']
 502
 503                 if changed:
 504                     print "%s rewound to %s" % (ref, change)
 505
 506         return True
 507
 508 class P4Submit(Command):
 509     def __init__(self):
 510         Command.__init__(self)
 511         self.options = [
 512                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 513                 optparse.make_option("--origin", dest="origin"),
 514                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 515         ]
 516         self.description = "Submit changes from git to the perforce depot."
 517         self.usage += " [name of git branch to submit into perforce depot]"
 518         self.interactive = True
 519         self.origin = ""
 520         self.detectRename = False
 521         self.verbose = False
 522         self.isWindows = (platform.system() == "Windows")
 523
 524     def check(self):
 525         if len(p4CmdList("opened ...")) > 0:
 526             die("You have files opened with perforce! Close them before starting the sync.")
 527
 528     # replaces everything between 'Description:' and the next P4 submit template field with the
 529     # commit message
 530     def prepareLogMessage(self, template, message):
 531         result = ""
 532
 533         inDescriptionSection = False
 534
 535         for line in template.split("\n"):
 536             if line.startswith("#"):
 537                 result += line + "\n"
 538                 continue
 539
 540             if inDescriptionSection:
 541                 if line.startswith("Files:"):
 542                     inDescriptionSection = False
 543                 else:
 544                     continue
 545             else:
 546                 if line.startswith("Description:"):
 547                     inDescriptionSection = True
 548                     line += "\n"
 549                     for messageLine in message.split("\n"):
 550                         line += "\t" + messageLine + "\n"
 551
 552             result += line + "\n"
 553
 554         return result
 555
 556     def prepareSubmitTemplate(self):
 557         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 558         template = ""
 559         inFilesSection = False
 560         for line in p4_read_pipe_lines("change -o"):
 561             if line.endswith("\r\n"):
 562                 line = line[:-2] + "\n"
 563             if inFilesSection:
 564                 if line.startswith("\t"):
 565                     # path starts and ends with a tab
 566                     path = line[1:]
 567                     lastTab = path.rfind("\t")
 568                     if lastTab != -1:
 569                         path = path[:lastTab]
 570                         if not path.startswith(self.depotPath):
 571                             continue
 572                 else:
 573                     inFilesSection = False
 574             else:
 575                 if line.startswith("Files:"):
 576                     inFilesSection = True
 577
 578             template += line
 579
 580         return template
 581
 582     def applyCommit(self, id):
 583         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 584         diffOpts = ("", "-M")[self.detectRename]
 585         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 586         filesToAdd = set()
 587         filesToDelete = set()
 588         editedFiles = set()
 589         filesToChangeExecBit = {}
 590         for line in diff:
 591             diff = parseDiffTreeEntry(line)
 592             modifier = diff['status']
 593             path = diff['src']
 594             if modifier == "M":
 595                 p4_system("edit \"%s\"" % path)
 596                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 597                     filesToChangeExecBit[path] = diff['dst_mode']
 598                 editedFiles.add(path)
 599             elif modifier == "A":
 600                 filesToAdd.add(path)
 601                 filesToChangeExecBit[path] = diff['dst_mode']
 602                 if path in filesToDelete:
 603                     filesToDelete.remove(path)
 604             elif modifier == "D":
 605                 filesToDelete.add(path)
 606                 if path in filesToAdd:
 607                     filesToAdd.remove(path)
 608             elif modifier == "R":
 609                 src, dest = diff['src'], diff['dst']
 610                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 611                 p4_system("edit \"%s\"" % (dest))
 612                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 613                     filesToChangeExecBit[dest] = diff['dst_mode']
 614                 os.unlink(dest)
 615                 editedFiles.add(dest)
 616                 filesToDelete.add(src)
 617             else:
 618                 die("unknown modifier %s for %s" % (modifier, path))
 619
 620         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 621         patchcmd = diffcmd + " | git apply "
 622         tryPatchCmd = patchcmd + "--check -"
 623         applyPatchCmd = patchcmd + "--check --apply -"
 624
 625         if os.system(tryPatchCmd) != 0:
 626             print "Unfortunately applying the change failed!"
 627             print "What do you want to do?"
 628             response = "x"
 629             while response != "s" and response != "a" and response != "w":
 630                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 631                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 632             if response == "s":
 633                 print "Skipping! Good luck with the next patches..."
 634                 for f in editedFiles:
 635                     p4_system("revert \"%s\"" % f);
 636                 for f in filesToAdd:
 637                     system("rm %s" %f)
 638                 return
 639             elif response == "a":
 640                 os.system(applyPatchCmd)
 641                 if len(filesToAdd) > 0:
 642                     print "You may also want to call p4 add on the following files:"
 643                     print " ".join(filesToAdd)
 644                 if len(filesToDelete):
 645                     print "The following files should be scheduled for deletion with p4 delete:"
 646                     print " ".join(filesToDelete)
 647                 die("Please resolve and submit the conflict manually and "
 648                     + "continue afterwards with git-p4 submit --continue")
 649             elif response == "w":
 650                 system(diffcmd + " > patch.txt")
 651                 print "Patch saved to patch.txt in %s !" % self.clientPath
 652                 die("Please resolve and submit the conflict manually and "
 653                     "continue afterwards with git-p4 submit --continue")
 654
 655         system(applyPatchCmd)
 656
 657         for f in filesToAdd:
 658             p4_system("add \"%s\"" % f)
 659         for f in filesToDelete:
 660             p4_system("revert \"%s\"" % f)
 661             p4_system("delete \"%s\"" % f)
 662
 663         # Set/clear executable bits
 664         for f in filesToChangeExecBit.keys():
 665             mode = filesToChangeExecBit[f]
 666             setP4ExecBit(f, mode)
 667
 668         logMessage = extractLogMessageFromGitCommit(id)
 669         logMessage = logMessage.strip()
 670
 671         template = self.prepareSubmitTemplate()
 672
 673         if self.interactive:
 674             submitTemplate = self.prepareLogMessage(template, logMessage)
 675             if os.environ.has_key("P4DIFF"):
 676                 del(os.environ["P4DIFF"])
 677             diff = read_pipe("p4 diff -du ...")
 678
 679             newdiff = ""
 680             for newFile in filesToAdd:
 681                 newdiff += "==== new file ====\n"
 682                 newdiff += "--- /dev/null\n"
 683                 newdiff += "+++ %s\n" % newFile
 684                 f = open(newFile, "r")
 685                 for line in f.readlines():
 686                     newdiff += "+" + line
 687                 f.close()
 688
 689             separatorLine = "######## everything below this line is just the diff #######\n"
 690
 691             [handle, fileName] = tempfile.mkstemp()
 692             tmpFile = os.fdopen(handle, "w+")
 693             if self.isWindows:
 694                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 695                 separatorLine = separatorLine.replace("\n", "\r\n")
 696                 newdiff = newdiff.replace("\n", "\r\n")
 697             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 698             tmpFile.close()
 699             defaultEditor = "vi"
 700             if platform.system() == "Windows":
 701                 defaultEditor = "notepad"
 702             if os.environ.has_key("P4EDITOR"):
 703                 editor = os.environ.get("P4EDITOR")
 704             else:
 705                 editor = os.environ.get("EDITOR", defaultEditor);
 706             system(editor + " " + fileName)
 707             tmpFile = open(fileName, "rb")
 708             message = tmpFile.read()
 709             tmpFile.close()
 710             os.remove(fileName)
 711             submitTemplate = message[:message.index(separatorLine)]
 712             if self.isWindows:
 713                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 714
 715             write_pipe("p4 submit -i", submitTemplate)
 716         else:
 717             fileName = "submit.txt"
 718             file = open(fileName, "w+")
 719             file.write(self.prepareLogMessage(template, logMessage))
 720             file.close()
 721             print ("Perforce submit template written as %s. "
 722                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 723                    % (fileName, fileName))
 724
 725     def run(self, args):
 726         if len(args) == 0:
 727             self.master = currentGitBranch()
 728             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 729                 die("Detecting current git branch failed!")
 730         elif len(args) == 1:
 731             self.master = args[0]
 732         else:
 733             return False
 734
 735         allowSubmit = gitConfig("git-p4.allowSubmit")
 736         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 737             die("%s is not in git-p4.allowSubmit" % self.master)
 738
 739         [upstream, settings] = findUpstreamBranchPoint()
 740         self.depotPath = settings['depot-paths'][0]
 741         if len(self.origin) == 0:
 742             self.origin = upstream
 743
 744         if self.verbose:
 745             print "Origin branch is " + self.origin
 746
 747         if len(self.depotPath) == 0:
 748             print "Internal error: cannot locate perforce depot path from existing branches"
 749             sys.exit(128)
 750
 751         self.clientPath = p4Where(self.depotPath)
 752
 753         if len(self.clientPath) == 0:
 754             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 755             sys.exit(128)
 756
 757         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 758         self.oldWorkingDirectory = os.getcwd()
 759
 760         os.chdir(self.clientPath)
 761         print "Syncronizing p4 checkout..."
 762         p4_system("sync ...")
 763
 764         self.check()
 765
 766         commits = []
 767         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 768             commits.append(line.strip())
 769         commits.reverse()
 770
 771         while len(commits) > 0:
 772             commit = commits[0]
 773             commits = commits[1:]
 774             self.applyCommit(commit)
 775             if not self.interactive:
 776                 break
 777
 778         if len(commits) == 0:
 779             print "All changes applied!"
 780             os.chdir(self.oldWorkingDirectory)
 781
 782             sync = P4Sync()
 783             sync.run([])
 784
 785             rebase = P4Rebase()
 786             rebase.rebase()
 787
 788         return True
 789
 790 class P4Sync(Command):
 791     def __init__(self):
 792         Command.__init__(self)
 793         self.options = [
 794                 optparse.make_option("--branch", dest="branch"),
 795                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 796                 optparse.make_option("--changesfile", dest="changesFile"),
 797                 optparse.make_option("--silent", dest="silent", action="store_true"),
 798                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 799                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 800                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 801                                      help="Import into refs/heads/ , not refs/remotes"),
 802                 optparse.make_option("--max-changes", dest="maxChanges"),
 803                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 804                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 805                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 806                                      help="Only sync files that are included in the Perforce Client Spec")
 807         ]
 808         self.description = """Imports from Perforce into a git repository.\n
 809     example:
 810     //depot/my/project/ -- to import the current head
 811     //depot/my/project/@all -- to import everything
 812     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 813
 814     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 815
 816         self.usage += " //depot/path[@revRange]"
 817         self.silent = False
 818         self.createdBranches = Set()
 819         self.committedChanges = Set()
 820         self.branch = ""
 821         self.detectBranches = False
 822         self.detectLabels = False
 823         self.changesFile = ""
 824         self.syncWithOrigin = True
 825         self.verbose = False
 826         self.importIntoRemotes = True
 827         self.maxChanges = ""
 828         self.isWindows = (platform.system() == "Windows")
 829         self.keepRepoPath = False
 830         self.depotPaths = None
 831         self.p4BranchesInGit = []
 832         self.cloneExclude = []
 833         self.useClientSpec = False
 834         self.clientSpecDirs = []
 835
 836         if gitConfig("git-p4.syncFromOrigin") == "false":
 837             self.syncWithOrigin = False
 838
 839     def extractFilesFromCommit(self, commit):
 840         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 841                              for path in self.cloneExclude]
 842         files = []
 843         fnum = 0
 844         while commit.has_key("depotFile%s" % fnum):
 845             path =  commit["depotFile%s" % fnum]
 846
 847             if [p for p in self.cloneExclude
 848                 if path.startswith (p)]:
 849                 found = False
 850             else:
 851                 found = [p for p in self.depotPaths
 852                          if path.startswith (p)]
 853             if not found:
 854                 fnum = fnum + 1
 855                 continue
 856
 857             file = {}
 858             file["path"] = path
 859             file["rev"] = commit["rev%s" % fnum]
 860             file["action"] = commit["action%s" % fnum]
 861             file["type"] = commit["type%s" % fnum]
 862             files.append(file)
 863             fnum = fnum + 1
 864         return files
 865
 866     def stripRepoPath(self, path, prefixes):
 867         if self.keepRepoPath:
 868             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 869
 870         for p in prefixes:
 871             if path.startswith(p):
 872                 path = path[len(p):]
 873
 874         return path
 875
 876     def splitFilesIntoBranches(self, commit):
 877         branches = {}
 878         fnum = 0
 879         while commit.has_key("depotFile%s" % fnum):
 880             path =  commit["depotFile%s" % fnum]
 881             found = [p for p in self.depotPaths
 882                      if path.startswith (p)]
 883             if not found:
 884                 fnum = fnum + 1
 885                 continue
 886
 887             file = {}
 888             file["path"] = path
 889             file["rev"] = commit["rev%s" % fnum]
 890             file["action"] = commit["action%s" % fnum]
 891             file["type"] = commit["type%s" % fnum]
 892             fnum = fnum + 1
 893
 894             relPath = self.stripRepoPath(path, self.depotPaths)
 895
 896             for branch in self.knownBranches.keys():
 897
 898                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 899                 if relPath.startswith(branch + "/"):
 900                     if branch not in branches:
 901                         branches[branch] = []
 902                     branches[branch].append(file)
 903                     break
 904
 905         return branches
 906
 907     ## Should move this out, doesn't use SELF.
 908     def readP4Files(self, files):
 909         filesForCommit = []
 910         filesToRead = []
 911
 912         for f in files:
 913             includeFile = True
 914             for val in self.clientSpecDirs:
 915                 if f['path'].startswith(val[0]):
 916                     if val[1] <= 0:
 917                         includeFile = False
 918                     break
 919
 920             if includeFile:
 921                 filesForCommit.append(f)
 922                 if f['action'] != 'delete':
 923                     filesToRead.append(f)
 924
 925         filedata = []
 926         if len(filesToRead) > 0:
 927             filedata = p4CmdList('-x - print',
 928                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 929                                                   for f in filesToRead]),
 930                                  stdin_mode='w+')
 931
 932             if "p4ExitCode" in filedata[0]:
 933                 die("Problems executing p4. Error: [%d]."
 934                     % (filedata[0]['p4ExitCode']));
 935
 936         j = 0;
 937         contents = {}
 938         while j < len(filedata):
 939             stat = filedata[j]
 940             j += 1
 941             text = [];
 942             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 943                 text.append(filedata[j]['data'])
 944                 j += 1
 945             text = ''.join(text)
 946
 947             if not stat.has_key('depotFile'):
 948                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 949                 continue
 950
 951             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 952                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 953             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 954                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 955
 956             contents[stat['depotFile']] = text
 957
 958         for f in filesForCommit:
 959             path = f['path']
 960             if contents.has_key(path):
 961                 f['data'] = contents[path]
 962
 963         return filesForCommit
 964
 965     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 966         epoch = details["time"]
 967         author = details["user"]
 968
 969         if self.verbose:
 970             print "commit into %s" % branch
 971
 972         # start with reading files; if that fails, we should not
 973         # create a commit.
 974         new_files = []
 975         for f in files:
 976             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 977                 new_files.append (f)
 978             else:
 979                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 980         files = self.readP4Files(new_files)
 981
 982         self.gitStream.write("commit %s\n" % branch)
 983 #        gitStream.write("mark :%s\n" % details["change"])
 984         self.committedChanges.add(int(details["change"]))
 985         committer = ""
 986         if author not in self.users:
 987             self.getUserMapFromPerforceServer()
 988         if author in self.users:
 989             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 990         else:
 991             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 992
 993         self.gitStream.write("committer %s\n" % committer)
 994
 995         self.gitStream.write("data <<EOT\n")
 996         self.gitStream.write(details["desc"])
 997         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 998                              % (','.join (branchPrefixes), details["change"]))
 999         if len(details['options']) > 0:
1000             self.gitStream.write(": options = %s" % details['options'])
1001         self.gitStream.write("]\nEOT\n\n")
1002
1003         if len(parent) > 0:
1004             if self.verbose:
1005                 print "parent %s" % parent
1006             self.gitStream.write("from %s\n" % parent)
1007
1008         for file in files:
1009             if file["type"] == "apple":
1010                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1011                 continue
1012
1013             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1014             if file["action"] == "delete":
1015                 self.gitStream.write("D %s\n" % relPath)
1016             else:
1017                 data = file['data']
1018
1019                 mode = "644"
1020                 if isP4Exec(file["type"]):
1021                     mode = "755"
1022                 elif file["type"] == "symlink":
1023                     mode = "120000"
1024                     # p4 print on a symlink contains "target\n", so strip it off
1025                     data = data[:-1]
1026
1027                 if self.isWindows and file["type"].endswith("text"):
1028                     data = data.replace("\r\n", "\n")
1029
1030                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1031                 self.gitStream.write("data %s\n" % len(data))
1032                 self.gitStream.write(data)
1033                 self.gitStream.write("\n")
1034
1035         self.gitStream.write("\n")
1036
1037         change = int(details["change"])
1038
1039         if self.labels.has_key(change):
1040             label = self.labels[change]
1041             labelDetails = label[0]
1042             labelRevisions = label[1]
1043             if self.verbose:
1044                 print "Change %s is labelled %s" % (change, labelDetails)
1045
1046             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1047                                                     for p in branchPrefixes]))
1048
1049             if len(files) == len(labelRevisions):
1050
1051                 cleanedFiles = {}
1052                 for info in files:
1053                     if info["action"] == "delete":
1054                         continue
1055                     cleanedFiles[info["depotFile"]] = info["rev"]
1056
1057                 if cleanedFiles == labelRevisions:
1058                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1059                     self.gitStream.write("from %s\n" % branch)
1060
1061                     owner = labelDetails["Owner"]
1062                     tagger = ""
1063                     if author in self.users:
1064                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1065                     else:
1066                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1067                     self.gitStream.write("tagger %s\n" % tagger)
1068                     self.gitStream.write("data <<EOT\n")
1069                     self.gitStream.write(labelDetails["Description"])
1070                     self.gitStream.write("EOT\n\n")
1071
1072                 else:
1073                     if not self.silent:
1074                         print ("Tag %s does not match with change %s: files do not match."
1075                                % (labelDetails["label"], change))
1076
1077             else:
1078                 if not self.silent:
1079                     print ("Tag %s does not match with change %s: file count is different."
1080                            % (labelDetails["label"], change))
1081
1082     def getUserCacheFilename(self):
1083         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1084         return home + "/.gitp4-usercache.txt"
1085
1086     def getUserMapFromPerforceServer(self):
1087         if self.userMapFromPerforceServer:
1088             return
1089         self.users = {}
1090
1091         for output in p4CmdList("users"):
1092             if not output.has_key("User"):
1093                 continue
1094             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1095
1096
1097         s = ''
1098         for (key, val) in self.users.items():
1099             s += "%s\t%s\n" % (key, val)
1100
1101         open(self.getUserCacheFilename(), "wb").write(s)
1102         self.userMapFromPerforceServer = True
1103
1104     def loadUserMapFromCache(self):
1105         self.users = {}
1106         self.userMapFromPerforceServer = False
1107         try:
1108             cache = open(self.getUserCacheFilename(), "rb")
1109             lines = cache.readlines()
1110             cache.close()
1111             for line in lines:
1112                 entry = line.strip().split("\t")
1113                 self.users[entry[0]] = entry[1]
1114         except IOError:
1115             self.getUserMapFromPerforceServer()
1116
1117     def getLabels(self):
1118         self.labels = {}
1119
1120         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1121         if len(l) > 0 and not self.silent:
1122             print "Finding files belonging to labels in %s" % `self.depotPaths`
1123
1124         for output in l:
1125             label = output["label"]
1126             revisions = {}
1127             newestChange = 0
1128             if self.verbose:
1129                 print "Querying files for label %s" % label
1130             for file in p4CmdList("files "
1131                                   +  ' '.join (["%s...@%s" % (p, label)
1132                                                 for p in self.depotPaths])):
1133                 revisions[file["depotFile"]] = file["rev"]
1134                 change = int(file["change"])
1135                 if change > newestChange:
1136                     newestChange = change
1137
1138             self.labels[newestChange] = [output, revisions]
1139
1140         if self.verbose:
1141             print "Label changes: %s" % self.labels.keys()
1142
1143     def guessProjectName(self):
1144         for p in self.depotPaths:
1145             if p.endswith("/"):
1146                 p = p[:-1]
1147             p = p[p.strip().rfind("/") + 1:]
1148             if not p.endswith("/"):
1149                p += "/"
1150             return p
1151
1152     def getBranchMapping(self):
1153         lostAndFoundBranches = set()
1154
1155         for info in p4CmdList("branches"):
1156             details = p4Cmd("branch -o %s" % info["branch"])
1157             viewIdx = 0
1158             while details.has_key("View%s" % viewIdx):
1159                 paths = details["View%s" % viewIdx].split(" ")
1160                 viewIdx = viewIdx + 1
1161                 # require standard //depot/foo/... //depot/bar/... mapping
1162                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1163                     continue
1164                 source = paths[0]
1165                 destination = paths[1]
1166                 ## HACK
1167                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1168                     source = source[len(self.depotPaths[0]):-4]
1169                     destination = destination[len(self.depotPaths[0]):-4]
1170
1171                     if destination in self.knownBranches:
1172                         if not self.silent:
1173                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1174                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1175                         continue
1176
1177                     self.knownBranches[destination] = source
1178
1179                     lostAndFoundBranches.discard(destination)
1180
1181                     if source not in self.knownBranches:
1182                         lostAndFoundBranches.add(source)
1183
1184
1185         for branch in lostAndFoundBranches:
1186             self.knownBranches[branch] = branch
1187
1188     def getBranchMappingFromGitBranches(self):
1189         branches = p4BranchesInGit(self.importIntoRemotes)
1190         for branch in branches.keys():
1191             if branch == "master":
1192                 branch = "main"
1193             else:
1194                 branch = branch[len(self.projectName):]
1195             self.knownBranches[branch] = branch
1196
1197     def listExistingP4GitBranches(self):
1198         # branches holds mapping from name to commit
1199         branches = p4BranchesInGit(self.importIntoRemotes)
1200         self.p4BranchesInGit = branches.keys()
1201         for branch in branches.keys():
1202             self.initialParents[self.refPrefix + branch] = branches[branch]
1203
1204     def updateOptionDict(self, d):
1205         option_keys = {}
1206         if self.keepRepoPath:
1207             option_keys['keepRepoPath'] = 1
1208
1209         d["options"] = ' '.join(sorted(option_keys.keys()))
1210
1211     def readOptions(self, d):
1212         self.keepRepoPath = (d.has_key('options')
1213                              and ('keepRepoPath' in d['options']))
1214
1215     def gitRefForBranch(self, branch):
1216         if branch == "main":
1217             return self.refPrefix + "master"
1218
1219         if len(branch) <= 0:
1220             return branch
1221
1222         return self.refPrefix + self.projectName + branch
1223
1224     def gitCommitByP4Change(self, ref, change):
1225         if self.verbose:
1226             print "looking in ref " + ref + " for change %s using bisect..." % change
1227
1228         earliestCommit = ""
1229         latestCommit = parseRevision(ref)
1230
1231         while True:
1232             if self.verbose:
1233                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1234             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1235             if len(next) == 0:
1236                 if self.verbose:
1237                     print "argh"
1238                 return ""
1239             log = extractLogMessageFromGitCommit(next)
1240             settings = extractSettingsGitLog(log)
1241             currentChange = int(settings['change'])
1242             if self.verbose:
1243                 print "current change %s" % currentChange
1244
1245             if currentChange == change:
1246                 if self.verbose:
1247                     print "found %s" % next
1248                 return next
1249
1250             if currentChange < change:
1251                 earliestCommit = "^%s" % next
1252             else:
1253                 latestCommit = "%s" % next
1254
1255         return ""
1256
1257     def importNewBranch(self, branch, maxChange):
1258         # make fast-import flush all changes to disk and update the refs using the checkpoint
1259         # command so that we can try to find the branch parent in the git history
1260         self.gitStream.write("checkpoint\n\n");
1261         self.gitStream.flush();
1262         branchPrefix = self.depotPaths[0] + branch + "/"
1263         range = "@1,%s" % maxChange
1264         #print "prefix" + branchPrefix
1265         changes = p4ChangesForPaths([branchPrefix], range)
1266         if len(changes) <= 0:
1267             return False
1268         firstChange = changes[0]
1269         #print "first change in branch: %s" % firstChange
1270         sourceBranch = self.knownBranches[branch]
1271         sourceDepotPath = self.depotPaths[0] + sourceBranch
1272         sourceRef = self.gitRefForBranch(sourceBranch)
1273         #print "source " + sourceBranch
1274
1275         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1276         #print "branch parent: %s" % branchParentChange
1277         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1278         if len(gitParent) > 0:
1279             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1280             #print "parent git commit: %s" % gitParent
1281
1282         self.importChanges(changes)
1283         return True
1284
1285     def importChanges(self, changes):
1286         cnt = 1
1287         for change in changes:
1288             description = p4Cmd("describe %s" % change)
1289             self.updateOptionDict(description)
1290
1291             if not self.silent:
1292                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1293                 sys.stdout.flush()
1294             cnt = cnt + 1
1295
1296             try:
1297                 if self.detectBranches:
1298                     branches = self.splitFilesIntoBranches(description)
1299                     for branch in branches.keys():
1300                         ## HACK  --hwn
1301                         branchPrefix = self.depotPaths[0] + branch + "/"
1302
1303                         parent = ""
1304
1305                         filesForCommit = branches[branch]
1306
1307                         if self.verbose:
1308                             print "branch is %s" % branch
1309
1310                         self.updatedBranches.add(branch)
1311
1312                         if branch not in self.createdBranches:
1313                             self.createdBranches.add(branch)
1314                             parent = self.knownBranches[branch]
1315                             if parent == branch:
1316                                 parent = ""
1317                             else:
1318                                 fullBranch = self.projectName + branch
1319                                 if fullBranch not in self.p4BranchesInGit:
1320                                     if not self.silent:
1321                                         print("\n    Importing new branch %s" % fullBranch);
1322                                     if self.importNewBranch(branch, change - 1):
1323                                         parent = ""
1324                                         self.p4BranchesInGit.append(fullBranch)
1325                                     if not self.silent:
1326                                         print("\n    Resuming with change %s" % change);
1327
1328                                 if self.verbose:
1329                                     print "parent determined through known branches: %s" % parent
1330
1331                         branch = self.gitRefForBranch(branch)
1332                         parent = self.gitRefForBranch(parent)
1333
1334                         if self.verbose:
1335                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1336
1337                         if len(parent) == 0 and branch in self.initialParents:
1338                             parent = self.initialParents[branch]
1339                             del self.initialParents[branch]
1340
1341                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1342                 else:
1343                     files = self.extractFilesFromCommit(description)
1344                     self.commit(description, files, self.branch, self.depotPaths,
1345                                 self.initialParent)
1346                     self.initialParent = ""
1347             except IOError:
1348                 print self.gitError.read()
1349                 sys.exit(1)
1350
1351     def importHeadRevision(self, revision):
1352         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1353
1354         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1355         details["desc"] = ("Initial import of %s from the state at revision %s"
1356                            % (' '.join(self.depotPaths), revision))
1357         details["change"] = revision
1358         newestRevision = 0
1359
1360         fileCnt = 0
1361         for info in p4CmdList("files "
1362                               +  ' '.join(["%s...%s"
1363                                            % (p, revision)
1364                                            for p in self.depotPaths])):
1365
1366             if info['code'] == 'error':
1367                 sys.stderr.write("p4 returned an error: %s\n"
1368                                  % info['data'])
1369                 sys.exit(1)
1370
1371
1372             change = int(info["change"])
1373             if change > newestRevision:
1374                 newestRevision = change
1375
1376             if info["action"] == "delete":
1377                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1378                 #fileCnt = fileCnt + 1
1379                 continue
1380
1381             for prop in ["depotFile", "rev", "action", "type" ]:
1382                 details["%s%s" % (prop, fileCnt)] = info[prop]
1383
1384             fileCnt = fileCnt + 1
1385
1386         details["change"] = newestRevision
1387         self.updateOptionDict(details)
1388         try:
1389             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1390         except IOError:
1391             print "IO error with git fast-import. Is your git version recent enough?"
1392             print self.gitError.read()
1393
1394
1395     def getClientSpec(self):
1396         specList = p4CmdList( "client -o" )
1397         temp = {}
1398         for entry in specList:
1399             for k,v in entry.iteritems():
1400                 if k.startswith("View"):
1401                     if v.startswith('"'):
1402                         start = 1
1403                     else:
1404                         start = 0
1405                     index = v.find("...")
1406                     v = v[start:index]
1407                     if v.startswith("-"):
1408                         v = v[1:]
1409                         temp[v] = -len(v)
1410                     else:
1411                         temp[v] = len(v)
1412         self.clientSpecDirs = temp.items()
1413         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1414
1415     def run(self, args):
1416         self.depotPaths = []
1417         self.changeRange = ""
1418         self.initialParent = ""
1419         self.previousDepotPaths = []
1420
1421         # map from branch depot path to parent branch
1422         self.knownBranches = {}
1423         self.initialParents = {}
1424         self.hasOrigin = originP4BranchesExist()
1425         if not self.syncWithOrigin:
1426             self.hasOrigin = False
1427
1428         if self.importIntoRemotes:
1429             self.refPrefix = "refs/remotes/p4/"
1430         else:
1431             self.refPrefix = "refs/heads/p4/"
1432
1433         if self.syncWithOrigin and self.hasOrigin:
1434             if not self.silent:
1435                 print "Syncing with origin first by calling git fetch origin"
1436             system("git fetch origin")
1437
1438         if len(self.branch) == 0:
1439             self.branch = self.refPrefix + "master"
1440             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1441                 system("git update-ref %s refs/heads/p4" % self.branch)
1442                 system("git branch -D p4");
1443             # create it /after/ importing, when master exists
1444             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1445                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1446
1447         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1448             self.getClientSpec()
1449
1450         # TODO: should always look at previous commits,
1451         # merge with previous imports, if possible.
1452         if args == []:
1453             if self.hasOrigin:
1454                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1455             self.listExistingP4GitBranches()
1456
1457             if len(self.p4BranchesInGit) > 1:
1458                 if not self.silent:
1459                     print "Importing from/into multiple branches"
1460                 self.detectBranches = True
1461
1462             if self.verbose:
1463                 print "branches: %s" % self.p4BranchesInGit
1464
1465             p4Change = 0
1466             for branch in self.p4BranchesInGit:
1467                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1468
1469                 settings = extractSettingsGitLog(logMsg)
1470
1471                 self.readOptions(settings)
1472                 if (settings.has_key('depot-paths')
1473                     and settings.has_key ('change')):
1474                     change = int(settings['change']) + 1
1475                     p4Change = max(p4Change, change)
1476
1477                     depotPaths = sorted(settings['depot-paths'])
1478                     if self.previousDepotPaths == []:
1479                         self.previousDepotPaths = depotPaths
1480                     else:
1481                         paths = []
1482                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1483                             for i in range(0, min(len(cur), len(prev))):
1484                                 if cur[i] <> prev[i]:
1485                                     i = i - 1
1486                                     break
1487
1488                             paths.append (cur[:i + 1])
1489
1490                         self.previousDepotPaths = paths
1491
1492             if p4Change > 0:
1493                 self.depotPaths = sorted(self.previousDepotPaths)
1494                 self.changeRange = "@%s,#head" % p4Change
1495                 if not self.detectBranches:
1496                     self.initialParent = parseRevision(self.branch)
1497                 if not self.silent and not self.detectBranches:
1498                     print "Performing incremental import into %s git branch" % self.branch
1499
1500         if not self.branch.startswith("refs/"):
1501             self.branch = "refs/heads/" + self.branch
1502
1503         if len(args) == 0 and self.depotPaths:
1504             if not self.silent:
1505                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1506         else:
1507             if self.depotPaths and self.depotPaths != args:
1508                 print ("previous import used depot path %s and now %s was specified. "
1509                        "This doesn't work!" % (' '.join (self.depotPaths),
1510                                                ' '.join (args)))
1511                 sys.exit(1)
1512
1513             self.depotPaths = sorted(args)
1514
1515         revision = ""
1516         self.users = {}
1517
1518         newPaths = []
1519         for p in self.depotPaths:
1520             if p.find("@") != -1:
1521                 atIdx = p.index("@")
1522                 self.changeRange = p[atIdx:]
1523                 if self.changeRange == "@all":
1524                     self.changeRange = ""
1525                 elif ',' not in self.changeRange:
1526                     revision = self.changeRange
1527                     self.changeRange = ""
1528                 p = p[:atIdx]
1529             elif p.find("#") != -1:
1530                 hashIdx = p.index("#")
1531                 revision = p[hashIdx:]
1532                 p = p[:hashIdx]
1533             elif self.previousDepotPaths == []:
1534                 revision = "#head"
1535
1536             p = re.sub ("\.\.\.$", "", p)
1537             if not p.endswith("/"):
1538                 p += "/"
1539
1540             newPaths.append(p)
1541
1542         self.depotPaths = newPaths
1543
1544
1545         self.loadUserMapFromCache()
1546         self.labels = {}
1547         if self.detectLabels:
1548             self.getLabels();
1549
1550         if self.detectBranches:
1551             ## FIXME - what's a P4 projectName ?
1552             self.projectName = self.guessProjectName()
1553
1554             if self.hasOrigin:
1555                 self.getBranchMappingFromGitBranches()
1556             else:
1557                 self.getBranchMapping()
1558             if self.verbose:
1559                 print "p4-git branches: %s" % self.p4BranchesInGit
1560                 print "initial parents: %s" % self.initialParents
1561             for b in self.p4BranchesInGit:
1562                 if b != "master":
1563
1564                     ## FIXME
1565                     b = b[len(self.projectName):]
1566                 self.createdBranches.add(b)
1567
1568         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1569
1570         importProcess = subprocess.Popen(["git", "fast-import"],
1571                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1572                                          stderr=subprocess.PIPE);
1573         self.gitOutput = importProcess.stdout
1574         self.gitStream = importProcess.stdin
1575         self.gitError = importProcess.stderr
1576
1577         if revision:
1578             self.importHeadRevision(revision)
1579         else:
1580             changes = []
1581
1582             if len(self.changesFile) > 0:
1583                 output = open(self.changesFile).readlines()
1584                 changeSet = Set()
1585                 for line in output:
1586                     changeSet.add(int(line))
1587
1588                 for change in changeSet:
1589                     changes.append(change)
1590
1591                 changes.sort()
1592             else:
1593                 if self.verbose:
1594                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1595                                                               self.changeRange)
1596                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1597
1598                 if len(self.maxChanges) > 0:
1599                     changes = changes[:min(int(self.maxChanges), len(changes))]
1600
1601             if len(changes) == 0:
1602                 if not self.silent:
1603                     print "No changes to import!"
1604                 return True
1605
1606             if not self.silent and not self.detectBranches:
1607                 print "Import destination: %s" % self.branch
1608
1609             self.updatedBranches = set()
1610
1611             self.importChanges(changes)
1612
1613             if not self.silent:
1614                 print ""
1615                 if len(self.updatedBranches) > 0:
1616                     sys.stdout.write("Updated branches: ")
1617                     for b in self.updatedBranches:
1618                         sys.stdout.write("%s " % b)
1619                     sys.stdout.write("\n")
1620
1621         self.gitStream.close()
1622         if importProcess.wait() != 0:
1623             die("fast-import failed: %s" % self.gitError.read())
1624         self.gitOutput.close()
1625         self.gitError.close()
1626
1627         return True
1628
1629 class P4Rebase(Command):
1630     def __init__(self):
1631         Command.__init__(self)
1632         self.options = [ ]
1633         self.description = ("Fetches the latest revision from perforce and "
1634                             + "rebases the current work (branch) against it")
1635         self.verbose = False
1636
1637     def run(self, args):
1638         sync = P4Sync()
1639         sync.run([])
1640
1641         return self.rebase()
1642
1643     def rebase(self):
1644         if os.system("git update-index --refresh") != 0:
1645             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1646         if len(read_pipe("git diff-index HEAD --")) > 0:
1647             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1648
1649         [upstream, settings] = findUpstreamBranchPoint()
1650         if len(upstream) == 0:
1651             die("Cannot find upstream branchpoint for rebase")
1652
1653         # the branchpoint may be p4/foo~3, so strip off the parent
1654         upstream = re.sub("~[0-9]+$", "", upstream)
1655
1656         print "Rebasing the current branch onto %s" % upstream
1657         oldHead = read_pipe("git rev-parse HEAD").strip()
1658         system("git rebase %s" % upstream)
1659         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1660         return True
1661
1662 class P4Clone(P4Sync):
1663     def __init__(self):
1664         P4Sync.__init__(self)
1665         self.description = "Creates a new git repository and imports from Perforce into it"
1666         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1667         self.options += [
1668             optparse.make_option("--destination", dest="cloneDestination",
1669                                  action='store', default=None,
1670                                  help="where to leave result of the clone"),
1671             optparse.make_option("-/", dest="cloneExclude",
1672                                  action="append", type="string",
1673                                  help="exclude depot path")
1674         ]
1675         self.cloneDestination = None
1676         self.needsGit = False
1677
1678     # This is required for the "append" cloneExclude action
1679     def ensure_value(self, attr, value):
1680         if not hasattr(self, attr) or getattr(self, attr) is None:
1681             setattr(self, attr, value)
1682         return getattr(self, attr)
1683
1684     def defaultDestination(self, args):
1685         ## TODO: use common prefix of args?
1686         depotPath = args[0]
1687         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1688         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1689         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1690         depotDir = re.sub(r"/$", "", depotDir)
1691         return os.path.split(depotDir)[1]
1692
1693     def run(self, args):
1694         if len(args) < 1:
1695             return False
1696
1697         if self.keepRepoPath and not self.cloneDestination:
1698             sys.stderr.write("Must specify destination for --keep-path\n")
1699             sys.exit(1)
1700
1701         depotPaths = args
1702
1703         if not self.cloneDestination and len(depotPaths) > 1:
1704             self.cloneDestination = depotPaths[-1]
1705             depotPaths = depotPaths[:-1]
1706
1707         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1708         for p in depotPaths:
1709             if not p.startswith("//"):
1710                 return False
1711
1712         if not self.cloneDestination:
1713             self.cloneDestination = self.defaultDestination(args)
1714
1715         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1716         if not os.path.exists(self.cloneDestination):
1717             os.makedirs(self.cloneDestination)
1718         os.chdir(self.cloneDestination)
1719         system("git init")
1720         self.gitdir = os.getcwd() + "/.git"
1721         if not P4Sync.run(self, depotPaths):
1722             return False
1723         if self.branch != "master":
1724             if gitBranchExists("refs/remotes/p4/master"):
1725                 system("git branch master refs/remotes/p4/master")
1726                 system("git checkout -f")
1727             else:
1728                 print "Could not detect main branch. No checkout/master branch created."
1729
1730         return True
1731
1732 class P4Branches(Command):
1733     def __init__(self):
1734         Command.__init__(self)
1735         self.options = [ ]
1736         self.description = ("Shows the git branches that hold imports and their "
1737                             + "corresponding perforce depot paths")
1738         self.verbose = False
1739
1740     def run(self, args):
1741         if originP4BranchesExist():
1742             createOrUpdateBranchesFromOrigin()
1743
1744         cmdline = "git rev-parse --symbolic "
1745         cmdline += " --remotes"
1746
1747         for line in read_pipe_lines(cmdline):
1748             line = line.strip()
1749
1750             if not line.startswith('p4/') or line == "p4/HEAD":
1751                 continue
1752             branch = line
1753
1754             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1755             settings = extractSettingsGitLog(log)
1756
1757             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1758         return True
1759
1760 class HelpFormatter(optparse.IndentedHelpFormatter):
1761     def __init__(self):
1762         optparse.IndentedHelpFormatter.__init__(self)
1763
1764     def format_description(self, description):
1765         if description:
1766             return description + "\n"
1767         else:
1768             return ""
1769
1770 def printUsage(commands):
1771     print "usage: %s <command> [options]" % sys.argv[0]
1772     print ""
1773     print "valid commands: %s" % ", ".join(commands)
1774     print ""
1775     print "Try %s <command> --help for command specific help." % sys.argv[0]
1776     print ""
1777
1778 commands = {
1779     "debug" : P4Debug,
1780     "submit" : P4Submit,
1781     "commit" : P4Submit,
1782     "sync" : P4Sync,
1783     "rebase" : P4Rebase,
1784     "clone" : P4Clone,
1785     "rollback" : P4RollBack,
1786     "branches" : P4Branches
1787 }
1788
1789
1790 def main():
1791     if len(sys.argv[1:]) == 0:
1792         printUsage(commands.keys())
1793         sys.exit(2)
1794
1795     cmd = ""
1796     cmdName = sys.argv[1]
1797     try:
1798         klass = commands[cmdName]
1799         cmd = klass()
1800     except KeyError:
1801         print "unknown command %s" % cmdName
1802         print ""
1803         printUsage(commands.keys())
1804         sys.exit(2)
1805
1806     options = cmd.options
1807     cmd.gitdir = os.environ.get("GIT_DIR", None)
1808
1809     args = sys.argv[2:]
1810
1811     if len(options) > 0:
1812         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1813
1814         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1815                                        options,
1816                                        description = cmd.description,
1817                                        formatter = HelpFormatter())
1818
1819         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1820     global verbose
1821     verbose = cmd.verbose
1822     if cmd.needsGit:
1823         if cmd.gitdir == None:
1824             cmd.gitdir = os.path.abspath(".git")
1825             if not isValidGitDir(cmd.gitdir):
1826                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1827                 if os.path.exists(cmd.gitdir):
1828                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1829                     if len(cdup) > 0:
1830                         os.chdir(cdup);
1831
1832         if not isValidGitDir(cmd.gitdir):
1833             if isValidGitDir(cmd.gitdir + "/.git"):
1834                 cmd.gitdir += "/.git"
1835             else:
1836                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1837
1838         os.environ["GIT_DIR"] = cmd.gitdir
1839
1840     if not cmd.run(args):
1841         parser.print_help()
1842
1843
1844 if __name__ == '__main__':
1845     main()