contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key, args = None): # set args to "--bool", for instance
 337     if not _gitConfig.has_key(key):
 338         argsFilter = ""
 339         if args != None:
 340             argsFilter = "%s " % args
 341         cmd = "git config %s%s" % (argsFilter, key)
 342         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 343     return _gitConfig[key]
 344
 345 def gitConfigList(key):
 346     if not _gitConfig.has_key(key):
 347         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 348     return _gitConfig[key]
 349
 350 def p4BranchesInGit(branchesAreInRemotes = True):
 351     branches = {}
 352
 353     cmdline = "git rev-parse --symbolic "
 354     if branchesAreInRemotes:
 355         cmdline += " --remotes"
 356     else:
 357         cmdline += " --branches"
 358
 359     for line in read_pipe_lines(cmdline):
 360         line = line.strip()
 361
 362         ## only import to p4/
 363         if not line.startswith('p4/') or line == "p4/HEAD":
 364             continue
 365         branch = line
 366
 367         # strip off p4
 368         branch = re.sub ("^p4/", "", line)
 369
 370         branches[branch] = parseRevision(line)
 371     return branches
 372
 373 def findUpstreamBranchPoint(head = "HEAD"):
 374     branches = p4BranchesInGit()
 375     # map from depot-path to branch name
 376     branchByDepotPath = {}
 377     for branch in branches.keys():
 378         tip = branches[branch]
 379         log = extractLogMessageFromGitCommit(tip)
 380         settings = extractSettingsGitLog(log)
 381         if settings.has_key("depot-paths"):
 382             paths = ",".join(settings["depot-paths"])
 383             branchByDepotPath[paths] = "remotes/p4/" + branch
 384
 385     settings = None
 386     parent = 0
 387     while parent < 65535:
 388         commit = head + "~%s" % parent
 389         log = extractLogMessageFromGitCommit(commit)
 390         settings = extractSettingsGitLog(log)
 391         if settings.has_key("depot-paths"):
 392             paths = ",".join(settings["depot-paths"])
 393             if branchByDepotPath.has_key(paths):
 394                 return [branchByDepotPath[paths], settings]
 395
 396         parent = parent + 1
 397
 398     return ["", settings]
 399
 400 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 401     if not silent:
 402         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 403                % localRefPrefix)
 404
 405     originPrefix = "origin/p4/"
 406
 407     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 408         line = line.strip()
 409         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 410             continue
 411
 412         headName = line[len(originPrefix):]
 413         remoteHead = localRefPrefix + headName
 414         originHead = line
 415
 416         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 417         if (not original.has_key('depot-paths')
 418             or not original.has_key('change')):
 419             continue
 420
 421         update = False
 422         if not gitBranchExists(remoteHead):
 423             if verbose:
 424                 print "creating %s" % remoteHead
 425             update = True
 426         else:
 427             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 428             if settings.has_key('change') > 0:
 429                 if settings['depot-paths'] == original['depot-paths']:
 430                     originP4Change = int(original['change'])
 431                     p4Change = int(settings['change'])
 432                     if originP4Change > p4Change:
 433                         print ("%s (%s) is newer than %s (%s). "
 434                                "Updating p4 branch from origin."
 435                                % (originHead, originP4Change,
 436                                   remoteHead, p4Change))
 437                         update = True
 438                 else:
 439                     print ("Ignoring: %s was imported from %s while "
 440                            "%s was imported from %s"
 441                            % (originHead, ','.join(original['depot-paths']),
 442                               remoteHead, ','.join(settings['depot-paths'])))
 443
 444         if update:
 445             system("git update-ref %s %s" % (remoteHead, originHead))
 446
 447 def originP4BranchesExist():
 448         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 449
 450 def p4ChangesForPaths(depotPaths, changeRange):
 451     assert depotPaths
 452     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 453                                                         for p in depotPaths]))
 454
 455     changes = {}
 456     for line in output:
 457         changeNum = int(line.split(" ")[1])
 458         changes[changeNum] = True
 459
 460     changelist = changes.keys()
 461     changelist.sort()
 462     return changelist
 463
 464 def p4PathStartsWith(path, prefix):
 465     # This method tries to remedy a potential mixed-case issue:
 466     #
 467     # If UserA adds  //depot/DirA/file1
 468     # and UserB adds //depot/dira/file2
 469     #
 470     # we may or may not have a problem. If you have core.ignorecase=true,
 471     # we treat DirA and dira as the same directory
 472     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 473     if ignorecase:
 474         return path.lower().startswith(prefix.lower())
 475     return path.startswith(prefix)
 476
 477 class Command:
 478     def __init__(self):
 479         self.usage = "usage: %prog [options]"
 480         self.needsGit = True
 481
 482 class P4UserMap:
 483     def __init__(self):
 484         self.userMapFromPerforceServer = False
 485
 486     def getUserCacheFilename(self):
 487         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 488         return home + "/.gitp4-usercache.txt"
 489
 490     def getUserMapFromPerforceServer(self):
 491         if self.userMapFromPerforceServer:
 492             return
 493         self.users = {}
 494         self.emails = {}
 495
 496         for output in p4CmdList("users"):
 497             if not output.has_key("User"):
 498                 continue
 499             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 500             self.emails[output["Email"]] = output["User"]
 501
 502
 503         s = ''
 504         for (key, val) in self.users.items():
 505             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 506
 507         open(self.getUserCacheFilename(), "wb").write(s)
 508         self.userMapFromPerforceServer = True
 509
 510     def loadUserMapFromCache(self):
 511         self.users = {}
 512         self.userMapFromPerforceServer = False
 513         try:
 514             cache = open(self.getUserCacheFilename(), "rb")
 515             lines = cache.readlines()
 516             cache.close()
 517             for line in lines:
 518                 entry = line.strip().split("\t")
 519                 self.users[entry[0]] = entry[1]
 520         except IOError:
 521             self.getUserMapFromPerforceServer()
 522
 523 class P4Debug(Command):
 524     def __init__(self):
 525         Command.__init__(self)
 526         self.options = [
 527             optparse.make_option("--verbose", dest="verbose", action="store_true",
 528                                  default=False),
 529             ]
 530         self.description = "A tool to debug the output of p4 -G."
 531         self.needsGit = False
 532         self.verbose = False
 533
 534     def run(self, args):
 535         j = 0
 536         for output in p4CmdList(" ".join(args)):
 537             print 'Element: %d' % j
 538             j += 1
 539             print output
 540         return True
 541
 542 class P4RollBack(Command):
 543     def __init__(self):
 544         Command.__init__(self)
 545         self.options = [
 546             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 547             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 548         ]
 549         self.description = "A tool to debug the multi-branch import. Don't use :)"
 550         self.verbose = False
 551         self.rollbackLocalBranches = False
 552
 553     def run(self, args):
 554         if len(args) != 1:
 555             return False
 556         maxChange = int(args[0])
 557
 558         if "p4ExitCode" in p4Cmd("changes -m 1"):
 559             die("Problems executing p4");
 560
 561         if self.rollbackLocalBranches:
 562             refPrefix = "refs/heads/"
 563             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 564         else:
 565             refPrefix = "refs/remotes/"
 566             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 567
 568         for line in lines:
 569             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 570                 line = line.strip()
 571                 ref = refPrefix + line
 572                 log = extractLogMessageFromGitCommit(ref)
 573                 settings = extractSettingsGitLog(log)
 574
 575                 depotPaths = settings['depot-paths']
 576                 change = settings['change']
 577
 578                 changed = False
 579
 580                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 581                                                            for p in depotPaths]))) == 0:
 582                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 583                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 584                     continue
 585
 586                 while change and int(change) > maxChange:
 587                     changed = True
 588                     if self.verbose:
 589                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 590                     system("git update-ref %s \"%s^\"" % (ref, ref))
 591                     log = extractLogMessageFromGitCommit(ref)
 592                     settings =  extractSettingsGitLog(log)
 593
 594
 595                     depotPaths = settings['depot-paths']
 596                     change = settings['change']
 597
 598                 if changed:
 599                     print "%s rewound to %s" % (ref, change)
 600
 601         return True
 602
 603 class P4Submit(Command, P4UserMap):
 604     def __init__(self):
 605         Command.__init__(self)
 606         P4UserMap.__init__(self)
 607         self.options = [
 608                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 609                 optparse.make_option("--origin", dest="origin"),
 610                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 611                 # preserve the user, requires relevant p4 permissions
 612                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 613         ]
 614         self.description = "Submit changes from git to the perforce depot."
 615         self.usage += " [name of git branch to submit into perforce depot]"
 616         self.interactive = True
 617         self.origin = ""
 618         self.detectRenames = False
 619         self.verbose = False
 620         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 621         self.isWindows = (platform.system() == "Windows")
 622         self.myP4UserId = None
 623
 624     def check(self):
 625         if len(p4CmdList("opened ...")) > 0:
 626             die("You have files opened with perforce! Close them before starting the sync.")
 627
 628     # replaces everything between 'Description:' and the next P4 submit template field with the
 629     # commit message
 630     def prepareLogMessage(self, template, message):
 631         result = ""
 632
 633         inDescriptionSection = False
 634
 635         for line in template.split("\n"):
 636             if line.startswith("#"):
 637                 result += line + "\n"
 638                 continue
 639
 640             if inDescriptionSection:
 641                 if line.startswith("Files:") or line.startswith("Jobs:"):
 642                     inDescriptionSection = False
 643                 else:
 644                     continue
 645             else:
 646                 if line.startswith("Description:"):
 647                     inDescriptionSection = True
 648                     line += "\n"
 649                     for messageLine in message.split("\n"):
 650                         line += "\t" + messageLine + "\n"
 651
 652             result += line + "\n"
 653
 654         return result
 655
 656     def p4UserForCommit(self,id):
 657         # Return the tuple (perforce user,git email) for a given git commit id
 658         self.getUserMapFromPerforceServer()
 659         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 660         gitEmail = gitEmail.strip()
 661         if not self.emails.has_key(gitEmail):
 662             return (None,gitEmail)
 663         else:
 664             return (self.emails[gitEmail],gitEmail)
 665
 666     def checkValidP4Users(self,commits):
 667         # check if any git authors cannot be mapped to p4 users
 668         for id in commits:
 669             (user,email) = self.p4UserForCommit(id)
 670             if not user:
 671                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 672                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 673                     print "%s" % msg
 674                 else:
 675                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 676
 677     def lastP4Changelist(self):
 678         # Get back the last changelist number submitted in this client spec. This
 679         # then gets used to patch up the username in the change. If the same
 680         # client spec is being used by multiple processes then this might go
 681         # wrong.
 682         results = p4CmdList("client -o")        # find the current client
 683         client = None
 684         for r in results:
 685             if r.has_key('Client'):
 686                 client = r['Client']
 687                 break
 688         if not client:
 689             die("could not get client spec")
 690         results = p4CmdList("changes -c %s -m 1" % client)
 691         for r in results:
 692             if r.has_key('change'):
 693                 return r['change']
 694         die("Could not get changelist number for last submit - cannot patch up user details")
 695
 696     def modifyChangelistUser(self, changelist, newUser):
 697         # fixup the user field of a changelist after it has been submitted.
 698         changes = p4CmdList("change -o %s" % changelist)
 699         if len(changes) != 1:
 700             die("Bad output from p4 change modifying %s to user %s" %
 701                 (changelist, newUser))
 702
 703         c = changes[0]
 704         if c['User'] == newUser: return   # nothing to do
 705         c['User'] = newUser
 706         input = marshal.dumps(c)
 707
 708         result = p4CmdList("change -f -i", stdin=input)
 709         for r in result:
 710             if r.has_key('code'):
 711                 if r['code'] == 'error':
 712                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 713             if r.has_key('data'):
 714                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 715                 return
 716         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 717
 718     def canChangeChangelists(self):
 719         # check to see if we have p4 admin or super-user permissions, either of
 720         # which are required to modify changelists.
 721         results = p4CmdList("protects %s" % self.depotPath)
 722         for r in results:
 723             if r.has_key('perm'):
 724                 if r['perm'] == 'admin':
 725                     return 1
 726                 if r['perm'] == 'super':
 727                     return 1
 728         return 0
 729
 730     def p4UserId(self):
 731         if self.myP4UserId:
 732             return self.myP4UserId
 733
 734         results = p4CmdList("user -o")
 735         for r in results:
 736             if r.has_key('User'):
 737                 self.myP4UserId = r['User']
 738                 return r['User']
 739         die("Could not find your p4 user id")
 740
 741     def p4UserIsMe(self, p4User):
 742         # return True if the given p4 user is actually me
 743         me = self.p4UserId()
 744         if not p4User or p4User != me:
 745             return False
 746         else:
 747             return True
 748
 749     def prepareSubmitTemplate(self):
 750         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 751         template = ""
 752         inFilesSection = False
 753         for line in p4_read_pipe_lines("change -o"):
 754             if line.endswith("\r\n"):
 755                 line = line[:-2] + "\n"
 756             if inFilesSection:
 757                 if line.startswith("\t"):
 758                     # path starts and ends with a tab
 759                     path = line[1:]
 760                     lastTab = path.rfind("\t")
 761                     if lastTab != -1:
 762                         path = path[:lastTab]
 763                         if not p4PathStartsWith(path, self.depotPath):
 764                             continue
 765                 else:
 766                     inFilesSection = False
 767             else:
 768                 if line.startswith("Files:"):
 769                     inFilesSection = True
 770
 771             template += line
 772
 773         return template
 774
 775     def applyCommit(self, id):
 776         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 777
 778         (p4User, gitEmail) = self.p4UserForCommit(id)
 779
 780         if not self.detectRenames:
 781             # If not explicitly set check the config variable
 782             self.detectRenames = gitConfig("git-p4.detectRenames")
 783
 784         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 785             diffOpts = ""
 786         elif self.detectRenames.lower() == "true":
 787             diffOpts = "-M"
 788         else:
 789             diffOpts = "-M%s" % self.detectRenames
 790
 791         detectCopies = gitConfig("git-p4.detectCopies")
 792         if detectCopies.lower() == "true":
 793             diffOpts += " -C"
 794         elif detectCopies != "" and detectCopies.lower() != "false":
 795             diffOpts += " -C%s" % detectCopies
 796
 797         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 798             diffOpts += " --find-copies-harder"
 799
 800         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 801         filesToAdd = set()
 802         filesToDelete = set()
 803         editedFiles = set()
 804         filesToChangeExecBit = {}
 805         for line in diff:
 806             diff = parseDiffTreeEntry(line)
 807             modifier = diff['status']
 808             path = diff['src']
 809             if modifier == "M":
 810                 p4_system("edit \"%s\"" % path)
 811                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 812                     filesToChangeExecBit[path] = diff['dst_mode']
 813                 editedFiles.add(path)
 814             elif modifier == "A":
 815                 filesToAdd.add(path)
 816                 filesToChangeExecBit[path] = diff['dst_mode']
 817                 if path in filesToDelete:
 818                     filesToDelete.remove(path)
 819             elif modifier == "D":
 820                 filesToDelete.add(path)
 821                 if path in filesToAdd:
 822                     filesToAdd.remove(path)
 823             elif modifier == "C":
 824                 src, dest = diff['src'], diff['dst']
 825                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 826                 if diff['src_sha1'] != diff['dst_sha1']:
 827                     p4_system("edit \"%s\"" % (dest))
 828                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 829                     p4_system("edit \"%s\"" % (dest))
 830                     filesToChangeExecBit[dest] = diff['dst_mode']
 831                 os.unlink(dest)
 832                 editedFiles.add(dest)
 833             elif modifier == "R":
 834                 src, dest = diff['src'], diff['dst']
 835                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 836                 if diff['src_sha1'] != diff['dst_sha1']:
 837                     p4_system("edit \"%s\"" % (dest))
 838                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 839                     p4_system("edit \"%s\"" % (dest))
 840                     filesToChangeExecBit[dest] = diff['dst_mode']
 841                 os.unlink(dest)
 842                 editedFiles.add(dest)
 843                 filesToDelete.add(src)
 844             else:
 845                 die("unknown modifier %s for %s" % (modifier, path))
 846
 847         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 848         patchcmd = diffcmd + " | git apply "
 849         tryPatchCmd = patchcmd + "--check -"
 850         applyPatchCmd = patchcmd + "--check --apply -"
 851
 852         if os.system(tryPatchCmd) != 0:
 853             print "Unfortunately applying the change failed!"
 854             print "What do you want to do?"
 855             response = "x"
 856             while response != "s" and response != "a" and response != "w":
 857                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 858                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 859             if response == "s":
 860                 print "Skipping! Good luck with the next patches..."
 861                 for f in editedFiles:
 862                     p4_system("revert \"%s\"" % f);
 863                 for f in filesToAdd:
 864                     system("rm %s" %f)
 865                 return
 866             elif response == "a":
 867                 os.system(applyPatchCmd)
 868                 if len(filesToAdd) > 0:
 869                     print "You may also want to call p4 add on the following files:"
 870                     print " ".join(filesToAdd)
 871                 if len(filesToDelete):
 872                     print "The following files should be scheduled for deletion with p4 delete:"
 873                     print " ".join(filesToDelete)
 874                 die("Please resolve and submit the conflict manually and "
 875                     + "continue afterwards with git-p4 submit --continue")
 876             elif response == "w":
 877                 system(diffcmd + " > patch.txt")
 878                 print "Patch saved to patch.txt in %s !" % self.clientPath
 879                 die("Please resolve and submit the conflict manually and "
 880                     "continue afterwards with git-p4 submit --continue")
 881
 882         system(applyPatchCmd)
 883
 884         for f in filesToAdd:
 885             p4_system("add \"%s\"" % f)
 886         for f in filesToDelete:
 887             p4_system("revert \"%s\"" % f)
 888             p4_system("delete \"%s\"" % f)
 889
 890         # Set/clear executable bits
 891         for f in filesToChangeExecBit.keys():
 892             mode = filesToChangeExecBit[f]
 893             setP4ExecBit(f, mode)
 894
 895         logMessage = extractLogMessageFromGitCommit(id)
 896         logMessage = logMessage.strip()
 897
 898         template = self.prepareSubmitTemplate()
 899
 900         if self.interactive:
 901             submitTemplate = self.prepareLogMessage(template, logMessage)
 902
 903             if self.preserveUser:
 904                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
 905
 906             if os.environ.has_key("P4DIFF"):
 907                 del(os.environ["P4DIFF"])
 908             diff = ""
 909             for editedFile in editedFiles:
 910                 diff += p4_read_pipe("diff -du %r" % editedFile)
 911
 912             newdiff = ""
 913             for newFile in filesToAdd:
 914                 newdiff += "==== new file ====\n"
 915                 newdiff += "--- /dev/null\n"
 916                 newdiff += "+++ %s\n" % newFile
 917                 f = open(newFile, "r")
 918                 for line in f.readlines():
 919                     newdiff += "+" + line
 920                 f.close()
 921
 922             if self.checkAuthorship and not self.p4UserIsMe(p4User):
 923                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
 924                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
 925                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
 926
 927             separatorLine = "######## everything below this line is just the diff #######\n"
 928
 929             [handle, fileName] = tempfile.mkstemp()
 930             tmpFile = os.fdopen(handle, "w+")
 931             if self.isWindows:
 932                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 933                 separatorLine = separatorLine.replace("\n", "\r\n")
 934                 newdiff = newdiff.replace("\n", "\r\n")
 935             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 936             tmpFile.close()
 937             mtime = os.stat(fileName).st_mtime
 938             if os.environ.has_key("P4EDITOR"):
 939                 editor = os.environ.get("P4EDITOR")
 940             else:
 941                 editor = read_pipe("git var GIT_EDITOR").strip()
 942             system(editor + " " + fileName)
 943
 944             if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 945                 checkModTime = False
 946             else:
 947                 checkModTime = True
 948
 949             response = "y"
 950             if checkModTime and (os.stat(fileName).st_mtime <= mtime):
 951                 response = "x"
 952                 while response != "y" and response != "n":
 953                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 954
 955             if response == "y":
 956                 tmpFile = open(fileName, "rb")
 957                 message = tmpFile.read()
 958                 tmpFile.close()
 959                 submitTemplate = message[:message.index(separatorLine)]
 960                 if self.isWindows:
 961                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 962                 p4_write_pipe("submit -i", submitTemplate)
 963
 964                 if self.preserveUser:
 965                     if p4User:
 966                         # Get last changelist number. Cannot easily get it from
 967                         # the submit command output as the output is unmarshalled.
 968                         changelist = self.lastP4Changelist()
 969                         self.modifyChangelistUser(changelist, p4User)
 970
 971             else:
 972                 for f in editedFiles:
 973                     p4_system("revert \"%s\"" % f);
 974                 for f in filesToAdd:
 975                     p4_system("revert \"%s\"" % f);
 976                     system("rm %s" %f)
 977
 978             os.remove(fileName)
 979         else:
 980             fileName = "submit.txt"
 981             file = open(fileName, "w+")
 982             file.write(self.prepareLogMessage(template, logMessage))
 983             file.close()
 984             print ("Perforce submit template written as %s. "
 985                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 986                    % (fileName, fileName))
 987
 988     def run(self, args):
 989         if len(args) == 0:
 990             self.master = currentGitBranch()
 991             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 992                 die("Detecting current git branch failed!")
 993         elif len(args) == 1:
 994             self.master = args[0]
 995         else:
 996             return False
 997
 998         allowSubmit = gitConfig("git-p4.allowSubmit")
 999         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1000             die("%s is not in git-p4.allowSubmit" % self.master)
1001
1002         [upstream, settings] = findUpstreamBranchPoint()
1003         self.depotPath = settings['depot-paths'][0]
1004         if len(self.origin) == 0:
1005             self.origin = upstream
1006
1007         if self.preserveUser:
1008             if not self.canChangeChangelists():
1009                 die("Cannot preserve user names without p4 super-user or admin permissions")
1010
1011         if self.verbose:
1012             print "Origin branch is " + self.origin
1013
1014         if len(self.depotPath) == 0:
1015             print "Internal error: cannot locate perforce depot path from existing branches"
1016             sys.exit(128)
1017
1018         self.clientPath = p4Where(self.depotPath)
1019
1020         if len(self.clientPath) == 0:
1021             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1022             sys.exit(128)
1023
1024         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1025         self.oldWorkingDirectory = os.getcwd()
1026
1027         chdir(self.clientPath)
1028         print "Synchronizing p4 checkout..."
1029         p4_system("sync ...")
1030
1031         self.check()
1032
1033         commits = []
1034         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1035             commits.append(line.strip())
1036         commits.reverse()
1037
1038         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1039             self.checkAuthorship = False
1040         else:
1041             self.checkAuthorship = True
1042
1043         if self.preserveUser:
1044             self.checkValidP4Users(commits)
1045
1046         while len(commits) > 0:
1047             commit = commits[0]
1048             commits = commits[1:]
1049             self.applyCommit(commit)
1050             if not self.interactive:
1051                 break
1052
1053         if len(commits) == 0:
1054             print "All changes applied!"
1055             chdir(self.oldWorkingDirectory)
1056
1057             sync = P4Sync()
1058             sync.run([])
1059
1060             rebase = P4Rebase()
1061             rebase.rebase()
1062
1063         return True
1064
1065 class P4Sync(Command, P4UserMap):
1066     delete_actions = ( "delete", "move/delete", "purge" )
1067
1068     def __init__(self):
1069         Command.__init__(self)
1070         P4UserMap.__init__(self)
1071         self.options = [
1072                 optparse.make_option("--branch", dest="branch"),
1073                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1074                 optparse.make_option("--changesfile", dest="changesFile"),
1075                 optparse.make_option("--silent", dest="silent", action="store_true"),
1076                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1077                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1078                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1079                                      help="Import into refs/heads/ , not refs/remotes"),
1080                 optparse.make_option("--max-changes", dest="maxChanges"),
1081                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1082                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1083                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1084                                      help="Only sync files that are included in the Perforce Client Spec")
1085         ]
1086         self.description = """Imports from Perforce into a git repository.\n
1087     example:
1088     //depot/my/project/ -- to import the current head
1089     //depot/my/project/@all -- to import everything
1090     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1091
1092     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1093
1094         self.usage += " //depot/path[@revRange]"
1095         self.silent = False
1096         self.createdBranches = set()
1097         self.committedChanges = set()
1098         self.branch = ""
1099         self.detectBranches = False
1100         self.detectLabels = False
1101         self.changesFile = ""
1102         self.syncWithOrigin = True
1103         self.verbose = False
1104         self.importIntoRemotes = True
1105         self.maxChanges = ""
1106         self.isWindows = (platform.system() == "Windows")
1107         self.keepRepoPath = False
1108         self.depotPaths = None
1109         self.p4BranchesInGit = []
1110         self.cloneExclude = []
1111         self.useClientSpec = False
1112         self.clientSpecDirs = []
1113
1114         if gitConfig("git-p4.syncFromOrigin") == "false":
1115             self.syncWithOrigin = False
1116
1117     #
1118     # P4 wildcards are not allowed in filenames.  P4 complains
1119     # if you simply add them, but you can force it with "-f", in
1120     # which case it translates them into %xx encoding internally.
1121     # Search for and fix just these four characters.  Do % last so
1122     # that fixing it does not inadvertently create new %-escapes.
1123     #
1124     def wildcard_decode(self, path):
1125         # Cannot have * in a filename in windows; untested as to
1126         # what p4 would do in such a case.
1127         if not self.isWindows:
1128             path = path.replace("%2A", "*")
1129         path = path.replace("%23", "#") \
1130                    .replace("%40", "@") \
1131                    .replace("%25", "%")
1132         return path
1133
1134     def extractFilesFromCommit(self, commit):
1135         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1136                              for path in self.cloneExclude]
1137         files = []
1138         fnum = 0
1139         while commit.has_key("depotFile%s" % fnum):
1140             path =  commit["depotFile%s" % fnum]
1141
1142             if [p for p in self.cloneExclude
1143                 if p4PathStartsWith(path, p)]:
1144                 found = False
1145             else:
1146                 found = [p for p in self.depotPaths
1147                          if p4PathStartsWith(path, p)]
1148             if not found:
1149                 fnum = fnum + 1
1150                 continue
1151
1152             file = {}
1153             file["path"] = path
1154             file["rev"] = commit["rev%s" % fnum]
1155             file["action"] = commit["action%s" % fnum]
1156             file["type"] = commit["type%s" % fnum]
1157             files.append(file)
1158             fnum = fnum + 1
1159         return files
1160
1161     def stripRepoPath(self, path, prefixes):
1162         if self.useClientSpec:
1163
1164             # if using the client spec, we use the output directory
1165             # specified in the client.  For example, a view
1166             #   //depot/foo/branch/... //client/branch/foo/...
1167             # will end up putting all foo/branch files into
1168             #  branch/foo/
1169             for val in self.clientSpecDirs:
1170                 if path.startswith(val[0]):
1171                     # replace the depot path with the client path
1172                     path = path.replace(val[0], val[1][1])
1173                     # now strip out the client (//client/...)
1174                     path = re.sub("^(//[^/]+/)", '', path)
1175                     # the rest is all path
1176                     return path
1177
1178         if self.keepRepoPath:
1179             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1180
1181         for p in prefixes:
1182             if p4PathStartsWith(path, p):
1183                 path = path[len(p):]
1184
1185         return path
1186
1187     def splitFilesIntoBranches(self, commit):
1188         branches = {}
1189         fnum = 0
1190         while commit.has_key("depotFile%s" % fnum):
1191             path =  commit["depotFile%s" % fnum]
1192             found = [p for p in self.depotPaths
1193                      if p4PathStartsWith(path, p)]
1194             if not found:
1195                 fnum = fnum + 1
1196                 continue
1197
1198             file = {}
1199             file["path"] = path
1200             file["rev"] = commit["rev%s" % fnum]
1201             file["action"] = commit["action%s" % fnum]
1202             file["type"] = commit["type%s" % fnum]
1203             fnum = fnum + 1
1204
1205             relPath = self.stripRepoPath(path, self.depotPaths)
1206
1207             for branch in self.knownBranches.keys():
1208
1209                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1210                 if relPath.startswith(branch + "/"):
1211                     if branch not in branches:
1212                         branches[branch] = []
1213                     branches[branch].append(file)
1214                     break
1215
1216         return branches
1217
1218     # output one file from the P4 stream
1219     # - helper for streamP4Files
1220
1221     def streamOneP4File(self, file, contents):
1222         if file["type"] == "apple":
1223             print "\nfile %s is a strange apple file that forks. Ignoring" % \
1224                 file['depotFile']
1225             return
1226
1227         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1228         relPath = self.wildcard_decode(relPath)
1229         if verbose:
1230             sys.stderr.write("%s\n" % relPath)
1231
1232         mode = "644"
1233         if isP4Exec(file["type"]):
1234             mode = "755"
1235         elif file["type"] == "symlink":
1236             mode = "120000"
1237             # p4 print on a symlink contains "target\n", so strip it off
1238             data = ''.join(contents)
1239             contents = [data[:-1]]
1240
1241         if file['type'].startswith("utf16"):
1242             # p4 delivers different text in the python output to -G
1243             # than it does when using "print -o", or normal p4 client
1244             # operations.  utf16 is converted to ascii or utf8, perhaps.
1245             # But ascii text saved as -t utf16 is completely mangled.
1246             # Invoke print -o to get the real contents.
1247             text = p4_read_pipe('print -q -o - "%s"' % file['depotFile'])
1248             contents = [ text ]
1249
1250         if self.isWindows and file["type"].endswith("text"):
1251             mangled = []
1252             for data in contents:
1253                 data = data.replace("\r\n", "\n")
1254                 mangled.append(data)
1255             contents = mangled
1256
1257         # Note that we do not try to de-mangle keywords on utf16 files,
1258         # even though in theory somebody may want that.
1259         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1260             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1261         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1262             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1263
1264         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1265
1266         # total length...
1267         length = 0
1268         for d in contents:
1269             length = length + len(d)
1270
1271         self.gitStream.write("data %d\n" % length)
1272         for d in contents:
1273             self.gitStream.write(d)
1274         self.gitStream.write("\n")
1275
1276     def streamOneP4Deletion(self, file):
1277         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1278         if verbose:
1279             sys.stderr.write("delete %s\n" % relPath)
1280         self.gitStream.write("D %s\n" % relPath)
1281
1282     # handle another chunk of streaming data
1283     def streamP4FilesCb(self, marshalled):
1284
1285         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1286             # start of a new file - output the old one first
1287             self.streamOneP4File(self.stream_file, self.stream_contents)
1288             self.stream_file = {}
1289             self.stream_contents = []
1290             self.stream_have_file_info = False
1291
1292         # pick up the new file information... for the
1293         # 'data' field we need to append to our array
1294         for k in marshalled.keys():
1295             if k == 'data':
1296                 self.stream_contents.append(marshalled['data'])
1297             else:
1298                 self.stream_file[k] = marshalled[k]
1299
1300         self.stream_have_file_info = True
1301
1302     # Stream directly from "p4 files" into "git fast-import"
1303     def streamP4Files(self, files):
1304         filesForCommit = []
1305         filesToRead = []
1306         filesToDelete = []
1307
1308         for f in files:
1309             includeFile = True
1310             for val in self.clientSpecDirs:
1311                 if f['path'].startswith(val[0]):
1312                     if val[1][0] <= 0:
1313                         includeFile = False
1314                     break
1315
1316             if includeFile:
1317                 filesForCommit.append(f)
1318                 if f['action'] in self.delete_actions:
1319                     filesToDelete.append(f)
1320                 else:
1321                     filesToRead.append(f)
1322
1323         # deleted files...
1324         for f in filesToDelete:
1325             self.streamOneP4Deletion(f)
1326
1327         if len(filesToRead) > 0:
1328             self.stream_file = {}
1329             self.stream_contents = []
1330             self.stream_have_file_info = False
1331
1332             # curry self argument
1333             def streamP4FilesCbSelf(entry):
1334                 self.streamP4FilesCb(entry)
1335
1336             p4CmdList("-x - print",
1337                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1338                                                   for f in filesToRead]),
1339                 cb=streamP4FilesCbSelf)
1340
1341             # do the last chunk
1342             if self.stream_file.has_key('depotFile'):
1343                 self.streamOneP4File(self.stream_file, self.stream_contents)
1344
1345     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1346         epoch = details["time"]
1347         author = details["user"]
1348         self.branchPrefixes = branchPrefixes
1349
1350         if self.verbose:
1351             print "commit into %s" % branch
1352
1353         # start with reading files; if that fails, we should not
1354         # create a commit.
1355         new_files = []
1356         for f in files:
1357             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1358                 new_files.append (f)
1359             else:
1360                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1361
1362         self.gitStream.write("commit %s\n" % branch)
1363 #        gitStream.write("mark :%s\n" % details["change"])
1364         self.committedChanges.add(int(details["change"]))
1365         committer = ""
1366         if author not in self.users:
1367             self.getUserMapFromPerforceServer()
1368         if author in self.users:
1369             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1370         else:
1371             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1372
1373         self.gitStream.write("committer %s\n" % committer)
1374
1375         self.gitStream.write("data <<EOT\n")
1376         self.gitStream.write(details["desc"])
1377         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1378                              % (','.join (branchPrefixes), details["change"]))
1379         if len(details['options']) > 0:
1380             self.gitStream.write(": options = %s" % details['options'])
1381         self.gitStream.write("]\nEOT\n\n")
1382
1383         if len(parent) > 0:
1384             if self.verbose:
1385                 print "parent %s" % parent
1386             self.gitStream.write("from %s\n" % parent)
1387
1388         self.streamP4Files(new_files)
1389         self.gitStream.write("\n")
1390
1391         change = int(details["change"])
1392
1393         if self.labels.has_key(change):
1394             label = self.labels[change]
1395             labelDetails = label[0]
1396             labelRevisions = label[1]
1397             if self.verbose:
1398                 print "Change %s is labelled %s" % (change, labelDetails)
1399
1400             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1401                                                     for p in branchPrefixes]))
1402
1403             if len(files) == len(labelRevisions):
1404
1405                 cleanedFiles = {}
1406                 for info in files:
1407                     if info["action"] in self.delete_actions:
1408                         continue
1409                     cleanedFiles[info["depotFile"]] = info["rev"]
1410
1411                 if cleanedFiles == labelRevisions:
1412                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1413                     self.gitStream.write("from %s\n" % branch)
1414
1415                     owner = labelDetails["Owner"]
1416                     tagger = ""
1417                     if author in self.users:
1418                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1419                     else:
1420                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1421                     self.gitStream.write("tagger %s\n" % tagger)
1422                     self.gitStream.write("data <<EOT\n")
1423                     self.gitStream.write(labelDetails["Description"])
1424                     self.gitStream.write("EOT\n\n")
1425
1426                 else:
1427                     if not self.silent:
1428                         print ("Tag %s does not match with change %s: files do not match."
1429                                % (labelDetails["label"], change))
1430
1431             else:
1432                 if not self.silent:
1433                     print ("Tag %s does not match with change %s: file count is different."
1434                            % (labelDetails["label"], change))
1435
1436     def getLabels(self):
1437         self.labels = {}
1438
1439         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1440         if len(l) > 0 and not self.silent:
1441             print "Finding files belonging to labels in %s" % `self.depotPaths`
1442
1443         for output in l:
1444             label = output["label"]
1445             revisions = {}
1446             newestChange = 0
1447             if self.verbose:
1448                 print "Querying files for label %s" % label
1449             for file in p4CmdList("files "
1450                                   +  ' '.join (["%s...@%s" % (p, label)
1451                                                 for p in self.depotPaths])):
1452                 revisions[file["depotFile"]] = file["rev"]
1453                 change = int(file["change"])
1454                 if change > newestChange:
1455                     newestChange = change
1456
1457             self.labels[newestChange] = [output, revisions]
1458
1459         if self.verbose:
1460             print "Label changes: %s" % self.labels.keys()
1461
1462     def guessProjectName(self):
1463         for p in self.depotPaths:
1464             if p.endswith("/"):
1465                 p = p[:-1]
1466             p = p[p.strip().rfind("/") + 1:]
1467             if not p.endswith("/"):
1468                p += "/"
1469             return p
1470
1471     def getBranchMapping(self):
1472         lostAndFoundBranches = set()
1473
1474         user = gitConfig("git-p4.branchUser")
1475         if len(user) > 0:
1476             command = "branches -u %s" % user
1477         else:
1478             command = "branches"
1479
1480         for info in p4CmdList(command):
1481             details = p4Cmd("branch -o %s" % info["branch"])
1482             viewIdx = 0
1483             while details.has_key("View%s" % viewIdx):
1484                 paths = details["View%s" % viewIdx].split(" ")
1485                 viewIdx = viewIdx + 1
1486                 # require standard //depot/foo/... //depot/bar/... mapping
1487                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1488                     continue
1489                 source = paths[0]
1490                 destination = paths[1]
1491                 ## HACK
1492                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1493                     source = source[len(self.depotPaths[0]):-4]
1494                     destination = destination[len(self.depotPaths[0]):-4]
1495
1496                     if destination in self.knownBranches:
1497                         if not self.silent:
1498                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1499                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1500                         continue
1501
1502                     self.knownBranches[destination] = source
1503
1504                     lostAndFoundBranches.discard(destination)
1505
1506                     if source not in self.knownBranches:
1507                         lostAndFoundBranches.add(source)
1508
1509         # Perforce does not strictly require branches to be defined, so we also
1510         # check git config for a branch list.
1511         #
1512         # Example of branch definition in git config file:
1513         # [git-p4]
1514         #   branchList=main:branchA
1515         #   branchList=main:branchB
1516         #   branchList=branchA:branchC
1517         configBranches = gitConfigList("git-p4.branchList")
1518         for branch in configBranches:
1519             if branch:
1520                 (source, destination) = branch.split(":")
1521                 self.knownBranches[destination] = source
1522
1523                 lostAndFoundBranches.discard(destination)
1524
1525                 if source not in self.knownBranches:
1526                     lostAndFoundBranches.add(source)
1527
1528
1529         for branch in lostAndFoundBranches:
1530             self.knownBranches[branch] = branch
1531
1532     def getBranchMappingFromGitBranches(self):
1533         branches = p4BranchesInGit(self.importIntoRemotes)
1534         for branch in branches.keys():
1535             if branch == "master":
1536                 branch = "main"
1537             else:
1538                 branch = branch[len(self.projectName):]
1539             self.knownBranches[branch] = branch
1540
1541     def listExistingP4GitBranches(self):
1542         # branches holds mapping from name to commit
1543         branches = p4BranchesInGit(self.importIntoRemotes)
1544         self.p4BranchesInGit = branches.keys()
1545         for branch in branches.keys():
1546             self.initialParents[self.refPrefix + branch] = branches[branch]
1547
1548     def updateOptionDict(self, d):
1549         option_keys = {}
1550         if self.keepRepoPath:
1551             option_keys['keepRepoPath'] = 1
1552
1553         d["options"] = ' '.join(sorted(option_keys.keys()))
1554
1555     def readOptions(self, d):
1556         self.keepRepoPath = (d.has_key('options')
1557                              and ('keepRepoPath' in d['options']))
1558
1559     def gitRefForBranch(self, branch):
1560         if branch == "main":
1561             return self.refPrefix + "master"
1562
1563         if len(branch) <= 0:
1564             return branch
1565
1566         return self.refPrefix + self.projectName + branch
1567
1568     def gitCommitByP4Change(self, ref, change):
1569         if self.verbose:
1570             print "looking in ref " + ref + " for change %s using bisect..." % change
1571
1572         earliestCommit = ""
1573         latestCommit = parseRevision(ref)
1574
1575         while True:
1576             if self.verbose:
1577                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1578             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1579             if len(next) == 0:
1580                 if self.verbose:
1581                     print "argh"
1582                 return ""
1583             log = extractLogMessageFromGitCommit(next)
1584             settings = extractSettingsGitLog(log)
1585             currentChange = int(settings['change'])
1586             if self.verbose:
1587                 print "current change %s" % currentChange
1588
1589             if currentChange == change:
1590                 if self.verbose:
1591                     print "found %s" % next
1592                 return next
1593
1594             if currentChange < change:
1595                 earliestCommit = "^%s" % next
1596             else:
1597                 latestCommit = "%s" % next
1598
1599         return ""
1600
1601     def importNewBranch(self, branch, maxChange):
1602         # make fast-import flush all changes to disk and update the refs using the checkpoint
1603         # command so that we can try to find the branch parent in the git history
1604         self.gitStream.write("checkpoint\n\n");
1605         self.gitStream.flush();
1606         branchPrefix = self.depotPaths[0] + branch + "/"
1607         range = "@1,%s" % maxChange
1608         #print "prefix" + branchPrefix
1609         changes = p4ChangesForPaths([branchPrefix], range)
1610         if len(changes) <= 0:
1611             return False
1612         firstChange = changes[0]
1613         #print "first change in branch: %s" % firstChange
1614         sourceBranch = self.knownBranches[branch]
1615         sourceDepotPath = self.depotPaths[0] + sourceBranch
1616         sourceRef = self.gitRefForBranch(sourceBranch)
1617         #print "source " + sourceBranch
1618
1619         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1620         #print "branch parent: %s" % branchParentChange
1621         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1622         if len(gitParent) > 0:
1623             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1624             #print "parent git commit: %s" % gitParent
1625
1626         self.importChanges(changes)
1627         return True
1628
1629     def importChanges(self, changes):
1630         cnt = 1
1631         for change in changes:
1632             description = p4Cmd("describe %s" % change)
1633             self.updateOptionDict(description)
1634
1635             if not self.silent:
1636                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1637                 sys.stdout.flush()
1638             cnt = cnt + 1
1639
1640             try:
1641                 if self.detectBranches:
1642                     branches = self.splitFilesIntoBranches(description)
1643                     for branch in branches.keys():
1644                         ## HACK  --hwn
1645                         branchPrefix = self.depotPaths[0] + branch + "/"
1646
1647                         parent = ""
1648
1649                         filesForCommit = branches[branch]
1650
1651                         if self.verbose:
1652                             print "branch is %s" % branch
1653
1654                         self.updatedBranches.add(branch)
1655
1656                         if branch not in self.createdBranches:
1657                             self.createdBranches.add(branch)
1658                             parent = self.knownBranches[branch]
1659                             if parent == branch:
1660                                 parent = ""
1661                             else:
1662                                 fullBranch = self.projectName + branch
1663                                 if fullBranch not in self.p4BranchesInGit:
1664                                     if not self.silent:
1665                                         print("\n    Importing new branch %s" % fullBranch);
1666                                     if self.importNewBranch(branch, change - 1):
1667                                         parent = ""
1668                                         self.p4BranchesInGit.append(fullBranch)
1669                                     if not self.silent:
1670                                         print("\n    Resuming with change %s" % change);
1671
1672                                 if self.verbose:
1673                                     print "parent determined through known branches: %s" % parent
1674
1675                         branch = self.gitRefForBranch(branch)
1676                         parent = self.gitRefForBranch(parent)
1677
1678                         if self.verbose:
1679                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1680
1681                         if len(parent) == 0 and branch in self.initialParents:
1682                             parent = self.initialParents[branch]
1683                             del self.initialParents[branch]
1684
1685                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1686                 else:
1687                     files = self.extractFilesFromCommit(description)
1688                     self.commit(description, files, self.branch, self.depotPaths,
1689                                 self.initialParent)
1690                     self.initialParent = ""
1691             except IOError:
1692                 print self.gitError.read()
1693                 sys.exit(1)
1694
1695     def importHeadRevision(self, revision):
1696         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1697
1698         details = {}
1699         details["user"] = "git perforce import user"
1700         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1701                            % (' '.join(self.depotPaths), revision))
1702         details["change"] = revision
1703         newestRevision = 0
1704
1705         fileCnt = 0
1706         for info in p4CmdList("files "
1707                               +  ' '.join(["%s...%s"
1708                                            % (p, revision)
1709                                            for p in self.depotPaths])):
1710
1711             if 'code' in info and info['code'] == 'error':
1712                 sys.stderr.write("p4 returned an error: %s\n"
1713                                  % info['data'])
1714                 if info['data'].find("must refer to client") >= 0:
1715                     sys.stderr.write("This particular p4 error is misleading.\n")
1716                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1717                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1718                 sys.exit(1)
1719             if 'p4ExitCode' in info:
1720                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1721                 sys.exit(1)
1722
1723
1724             change = int(info["change"])
1725             if change > newestRevision:
1726                 newestRevision = change
1727
1728             if info["action"] in self.delete_actions:
1729                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1730                 #fileCnt = fileCnt + 1
1731                 continue
1732
1733             for prop in ["depotFile", "rev", "action", "type" ]:
1734                 details["%s%s" % (prop, fileCnt)] = info[prop]
1735
1736             fileCnt = fileCnt + 1
1737
1738         details["change"] = newestRevision
1739
1740         # Use time from top-most change so that all git-p4 clones of
1741         # the same p4 repo have the same commit SHA1s.
1742         res = p4CmdList("describe -s %d" % newestRevision)
1743         newestTime = None
1744         for r in res:
1745             if r.has_key('time'):
1746                 newestTime = int(r['time'])
1747         if newestTime is None:
1748             die("\"describe -s\" on newest change %d did not give a time")
1749         details["time"] = newestTime
1750
1751         self.updateOptionDict(details)
1752         try:
1753             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1754         except IOError:
1755             print "IO error with git fast-import. Is your git version recent enough?"
1756             print self.gitError.read()
1757
1758
1759     def getClientSpec(self):
1760         specList = p4CmdList( "client -o" )
1761         temp = {}
1762         for entry in specList:
1763             for k,v in entry.iteritems():
1764                 if k.startswith("View"):
1765
1766                     # p4 has these %%1 to %%9 arguments in specs to
1767                     # reorder paths; which we can't handle (yet :)
1768                     if re.match('%%\d', v) != None:
1769                         print "Sorry, can't handle %%n arguments in client specs"
1770                         sys.exit(1)
1771
1772                     if v.startswith('"'):
1773                         start = 1
1774                     else:
1775                         start = 0
1776                     index = v.find("...")
1777
1778                     # save the "client view"; i.e the RHS of the view
1779                     # line that tells the client where to put the
1780                     # files for this view.
1781                     cv = v[index+3:].strip() # +3 to remove previous '...'
1782
1783                     # if the client view doesn't end with a
1784                     # ... wildcard, then we're going to mess up the
1785                     # output directory, so fail gracefully.
1786                     if not cv.endswith('...'):
1787                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1788                         sys.exit(1)
1789                     cv=cv[:-3]
1790
1791                     # now save the view; +index means included, -index
1792                     # means it should be filtered out.
1793                     v = v[start:index]
1794                     if v.startswith("-"):
1795                         v = v[1:]
1796                         include = -len(v)
1797                     else:
1798                         include = len(v)
1799
1800                     temp[v] = (include, cv)
1801
1802         self.clientSpecDirs = temp.items()
1803         self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1804
1805     def run(self, args):
1806         self.depotPaths = []
1807         self.changeRange = ""
1808         self.initialParent = ""
1809         self.previousDepotPaths = []
1810
1811         # map from branch depot path to parent branch
1812         self.knownBranches = {}
1813         self.initialParents = {}
1814         self.hasOrigin = originP4BranchesExist()
1815         if not self.syncWithOrigin:
1816             self.hasOrigin = False
1817
1818         if self.importIntoRemotes:
1819             self.refPrefix = "refs/remotes/p4/"
1820         else:
1821             self.refPrefix = "refs/heads/p4/"
1822
1823         if self.syncWithOrigin and self.hasOrigin:
1824             if not self.silent:
1825                 print "Syncing with origin first by calling git fetch origin"
1826             system("git fetch origin")
1827
1828         if len(self.branch) == 0:
1829             self.branch = self.refPrefix + "master"
1830             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1831                 system("git update-ref %s refs/heads/p4" % self.branch)
1832                 system("git branch -D p4");
1833             # create it /after/ importing, when master exists
1834             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1835                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1836
1837         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1838             self.getClientSpec()
1839
1840         # TODO: should always look at previous commits,
1841         # merge with previous imports, if possible.
1842         if args == []:
1843             if self.hasOrigin:
1844                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1845             self.listExistingP4GitBranches()
1846
1847             if len(self.p4BranchesInGit) > 1:
1848                 if not self.silent:
1849                     print "Importing from/into multiple branches"
1850                 self.detectBranches = True
1851
1852             if self.verbose:
1853                 print "branches: %s" % self.p4BranchesInGit
1854
1855             p4Change = 0
1856             for branch in self.p4BranchesInGit:
1857                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1858
1859                 settings = extractSettingsGitLog(logMsg)
1860
1861                 self.readOptions(settings)
1862                 if (settings.has_key('depot-paths')
1863                     and settings.has_key ('change')):
1864                     change = int(settings['change']) + 1
1865                     p4Change = max(p4Change, change)
1866
1867                     depotPaths = sorted(settings['depot-paths'])
1868                     if self.previousDepotPaths == []:
1869                         self.previousDepotPaths = depotPaths
1870                     else:
1871                         paths = []
1872                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1873                             prev_list = prev.split("/")
1874                             cur_list = cur.split("/")
1875                             for i in range(0, min(len(cur_list), len(prev_list))):
1876                                 if cur_list[i] <> prev_list[i]:
1877                                     i = i - 1
1878                                     break
1879
1880                             paths.append ("/".join(cur_list[:i + 1]))
1881
1882                         self.previousDepotPaths = paths
1883
1884             if p4Change > 0:
1885                 self.depotPaths = sorted(self.previousDepotPaths)
1886                 self.changeRange = "@%s,#head" % p4Change
1887                 if not self.detectBranches:
1888                     self.initialParent = parseRevision(self.branch)
1889                 if not self.silent and not self.detectBranches:
1890                     print "Performing incremental import into %s git branch" % self.branch
1891
1892         if not self.branch.startswith("refs/"):
1893             self.branch = "refs/heads/" + self.branch
1894
1895         if len(args) == 0 and self.depotPaths:
1896             if not self.silent:
1897                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1898         else:
1899             if self.depotPaths and self.depotPaths != args:
1900                 print ("previous import used depot path %s and now %s was specified. "
1901                        "This doesn't work!" % (' '.join (self.depotPaths),
1902                                                ' '.join (args)))
1903                 sys.exit(1)
1904
1905             self.depotPaths = sorted(args)
1906
1907         revision = ""
1908         self.users = {}
1909
1910         newPaths = []
1911         for p in self.depotPaths:
1912             if p.find("@") != -1:
1913                 atIdx = p.index("@")
1914                 self.changeRange = p[atIdx:]
1915                 if self.changeRange == "@all":
1916                     self.changeRange = ""
1917                 elif ',' not in self.changeRange:
1918                     revision = self.changeRange
1919                     self.changeRange = ""
1920                 p = p[:atIdx]
1921             elif p.find("#") != -1:
1922                 hashIdx = p.index("#")
1923                 revision = p[hashIdx:]
1924                 p = p[:hashIdx]
1925             elif self.previousDepotPaths == []:
1926                 revision = "#head"
1927
1928             p = re.sub ("\.\.\.$", "", p)
1929             if not p.endswith("/"):
1930                 p += "/"
1931
1932             newPaths.append(p)
1933
1934         self.depotPaths = newPaths
1935
1936
1937         self.loadUserMapFromCache()
1938         self.labels = {}
1939         if self.detectLabels:
1940             self.getLabels();
1941
1942         if self.detectBranches:
1943             ## FIXME - what's a P4 projectName ?
1944             self.projectName = self.guessProjectName()
1945
1946             if self.hasOrigin:
1947                 self.getBranchMappingFromGitBranches()
1948             else:
1949                 self.getBranchMapping()
1950             if self.verbose:
1951                 print "p4-git branches: %s" % self.p4BranchesInGit
1952                 print "initial parents: %s" % self.initialParents
1953             for b in self.p4BranchesInGit:
1954                 if b != "master":
1955
1956                     ## FIXME
1957                     b = b[len(self.projectName):]
1958                 self.createdBranches.add(b)
1959
1960         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1961
1962         importProcess = subprocess.Popen(["git", "fast-import"],
1963                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1964                                          stderr=subprocess.PIPE);
1965         self.gitOutput = importProcess.stdout
1966         self.gitStream = importProcess.stdin
1967         self.gitError = importProcess.stderr
1968
1969         if revision:
1970             self.importHeadRevision(revision)
1971         else:
1972             changes = []
1973
1974             if len(self.changesFile) > 0:
1975                 output = open(self.changesFile).readlines()
1976                 changeSet = set()
1977                 for line in output:
1978                     changeSet.add(int(line))
1979
1980                 for change in changeSet:
1981                     changes.append(change)
1982
1983                 changes.sort()
1984             else:
1985                 # catch "git-p4 sync" with no new branches, in a repo that
1986                 # does not have any existing git-p4 branches
1987                 if len(args) == 0 and not self.p4BranchesInGit:
1988                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
1989                 if self.verbose:
1990                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1991                                                               self.changeRange)
1992                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1993
1994                 if len(self.maxChanges) > 0:
1995                     changes = changes[:min(int(self.maxChanges), len(changes))]
1996
1997             if len(changes) == 0:
1998                 if not self.silent:
1999                     print "No changes to import!"
2000                 return True
2001
2002             if not self.silent and not self.detectBranches:
2003                 print "Import destination: %s" % self.branch
2004
2005             self.updatedBranches = set()
2006
2007             self.importChanges(changes)
2008
2009             if not self.silent:
2010                 print ""
2011                 if len(self.updatedBranches) > 0:
2012                     sys.stdout.write("Updated branches: ")
2013                     for b in self.updatedBranches:
2014                         sys.stdout.write("%s " % b)
2015                     sys.stdout.write("\n")
2016
2017         self.gitStream.close()
2018         if importProcess.wait() != 0:
2019             die("fast-import failed: %s" % self.gitError.read())
2020         self.gitOutput.close()
2021         self.gitError.close()
2022
2023         return True
2024
2025 class P4Rebase(Command):
2026     def __init__(self):
2027         Command.__init__(self)
2028         self.options = [ ]
2029         self.description = ("Fetches the latest revision from perforce and "
2030                             + "rebases the current work (branch) against it")
2031         self.verbose = False
2032
2033     def run(self, args):
2034         sync = P4Sync()
2035         sync.run([])
2036
2037         return self.rebase()
2038
2039     def rebase(self):
2040         if os.system("git update-index --refresh") != 0:
2041             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2042         if len(read_pipe("git diff-index HEAD --")) > 0:
2043             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2044
2045         [upstream, settings] = findUpstreamBranchPoint()
2046         if len(upstream) == 0:
2047             die("Cannot find upstream branchpoint for rebase")
2048
2049         # the branchpoint may be p4/foo~3, so strip off the parent
2050         upstream = re.sub("~[0-9]+$", "", upstream)
2051
2052         print "Rebasing the current branch onto %s" % upstream
2053         oldHead = read_pipe("git rev-parse HEAD").strip()
2054         system("git rebase %s" % upstream)
2055         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2056         return True
2057
2058 class P4Clone(P4Sync):
2059     def __init__(self):
2060         P4Sync.__init__(self)
2061         self.description = "Creates a new git repository and imports from Perforce into it"
2062         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2063         self.options += [
2064             optparse.make_option("--destination", dest="cloneDestination",
2065                                  action='store', default=None,
2066                                  help="where to leave result of the clone"),
2067             optparse.make_option("-/", dest="cloneExclude",
2068                                  action="append", type="string",
2069                                  help="exclude depot path"),
2070             optparse.make_option("--bare", dest="cloneBare",
2071                                  action="store_true", default=False),
2072         ]
2073         self.cloneDestination = None
2074         self.needsGit = False
2075         self.cloneBare = False
2076
2077     # This is required for the "append" cloneExclude action
2078     def ensure_value(self, attr, value):
2079         if not hasattr(self, attr) or getattr(self, attr) is None:
2080             setattr(self, attr, value)
2081         return getattr(self, attr)
2082
2083     def defaultDestination(self, args):
2084         ## TODO: use common prefix of args?
2085         depotPath = args[0]
2086         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2087         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2088         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2089         depotDir = re.sub(r"/$", "", depotDir)
2090         return os.path.split(depotDir)[1]
2091
2092     def run(self, args):
2093         if len(args) < 1:
2094             return False
2095
2096         if self.keepRepoPath and not self.cloneDestination:
2097             sys.stderr.write("Must specify destination for --keep-path\n")
2098             sys.exit(1)
2099
2100         depotPaths = args
2101
2102         if not self.cloneDestination and len(depotPaths) > 1:
2103             self.cloneDestination = depotPaths[-1]
2104             depotPaths = depotPaths[:-1]
2105
2106         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2107         for p in depotPaths:
2108             if not p.startswith("//"):
2109                 return False
2110
2111         if not self.cloneDestination:
2112             self.cloneDestination = self.defaultDestination(args)
2113
2114         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2115
2116         if not os.path.exists(self.cloneDestination):
2117             os.makedirs(self.cloneDestination)
2118         chdir(self.cloneDestination)
2119
2120         init_cmd = [ "git", "init" ]
2121         if self.cloneBare:
2122             init_cmd.append("--bare")
2123         subprocess.check_call(init_cmd)
2124
2125         if not P4Sync.run(self, depotPaths):
2126             return False
2127         if self.branch != "master":
2128             if self.importIntoRemotes:
2129                 masterbranch = "refs/remotes/p4/master"
2130             else:
2131                 masterbranch = "refs/heads/p4/master"
2132             if gitBranchExists(masterbranch):
2133                 system("git branch master %s" % masterbranch)
2134                 if not self.cloneBare:
2135                     system("git checkout -f")
2136             else:
2137                 print "Could not detect main branch. No checkout/master branch created."
2138
2139         return True
2140
2141 class P4Branches(Command):
2142     def __init__(self):
2143         Command.__init__(self)
2144         self.options = [ ]
2145         self.description = ("Shows the git branches that hold imports and their "
2146                             + "corresponding perforce depot paths")
2147         self.verbose = False
2148
2149     def run(self, args):
2150         if originP4BranchesExist():
2151             createOrUpdateBranchesFromOrigin()
2152
2153         cmdline = "git rev-parse --symbolic "
2154         cmdline += " --remotes"
2155
2156         for line in read_pipe_lines(cmdline):
2157             line = line.strip()
2158
2159             if not line.startswith('p4/') or line == "p4/HEAD":
2160                 continue
2161             branch = line
2162
2163             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2164             settings = extractSettingsGitLog(log)
2165
2166             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2167         return True
2168
2169 class HelpFormatter(optparse.IndentedHelpFormatter):
2170     def __init__(self):
2171         optparse.IndentedHelpFormatter.__init__(self)
2172
2173     def format_description(self, description):
2174         if description:
2175             return description + "\n"
2176         else:
2177             return ""
2178
2179 def printUsage(commands):
2180     print "usage: %s <command> [options]" % sys.argv[0]
2181     print ""
2182     print "valid commands: %s" % ", ".join(commands)
2183     print ""
2184     print "Try %s <command> --help for command specific help." % sys.argv[0]
2185     print ""
2186
2187 commands = {
2188     "debug" : P4Debug,
2189     "submit" : P4Submit,
2190     "commit" : P4Submit,
2191     "sync" : P4Sync,
2192     "rebase" : P4Rebase,
2193     "clone" : P4Clone,
2194     "rollback" : P4RollBack,
2195     "branches" : P4Branches
2196 }
2197
2198
2199 def main():
2200     if len(sys.argv[1:]) == 0:
2201         printUsage(commands.keys())
2202         sys.exit(2)
2203
2204     cmd = ""
2205     cmdName = sys.argv[1]
2206     try:
2207         klass = commands[cmdName]
2208         cmd = klass()
2209     except KeyError:
2210         print "unknown command %s" % cmdName
2211         print ""
2212         printUsage(commands.keys())
2213         sys.exit(2)
2214
2215     options = cmd.options
2216     cmd.gitdir = os.environ.get("GIT_DIR", None)
2217
2218     args = sys.argv[2:]
2219
2220     if len(options) > 0:
2221         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2222
2223         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2224                                        options,
2225                                        description = cmd.description,
2226                                        formatter = HelpFormatter())
2227
2228         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2229     global verbose
2230     verbose = cmd.verbose
2231     if cmd.needsGit:
2232         if cmd.gitdir == None:
2233             cmd.gitdir = os.path.abspath(".git")
2234             if not isValidGitDir(cmd.gitdir):
2235                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2236                 if os.path.exists(cmd.gitdir):
2237                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2238                     if len(cdup) > 0:
2239                         chdir(cdup);
2240
2241         if not isValidGitDir(cmd.gitdir):
2242             if isValidGitDir(cmd.gitdir + "/.git"):
2243                 cmd.gitdir += "/.git"
2244             else:
2245                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2246
2247         os.environ["GIT_DIR"] = cmd.gitdir
2248
2249     if not cmd.run(args):
2250         parser.print_help()
2251
2252
2253 if __name__ == '__main__':
2254     main()