contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key):
 337     if not _gitConfig.has_key(key):
 338         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339     return _gitConfig[key]
 340
 341 def p4BranchesInGit(branchesAreInRemotes = True):
 342     branches = {}
 343
 344     cmdline = "git rev-parse --symbolic "
 345     if branchesAreInRemotes:
 346         cmdline += " --remotes"
 347     else:
 348         cmdline += " --branches"
 349
 350     for line in read_pipe_lines(cmdline):
 351         line = line.strip()
 352
 353         ## only import to p4/
 354         if not line.startswith('p4/') or line == "p4/HEAD":
 355             continue
 356         branch = line
 357
 358         # strip off p4
 359         branch = re.sub ("^p4/", "", line)
 360
 361         branches[branch] = parseRevision(line)
 362     return branches
 363
 364 def findUpstreamBranchPoint(head = "HEAD"):
 365     branches = p4BranchesInGit()
 366     # map from depot-path to branch name
 367     branchByDepotPath = {}
 368     for branch in branches.keys():
 369         tip = branches[branch]
 370         log = extractLogMessageFromGitCommit(tip)
 371         settings = extractSettingsGitLog(log)
 372         if settings.has_key("depot-paths"):
 373             paths = ",".join(settings["depot-paths"])
 374             branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376     settings = None
 377     parent = 0
 378     while parent < 65535:
 379         commit = head + "~%s" % parent
 380         log = extractLogMessageFromGitCommit(commit)
 381         settings = extractSettingsGitLog(log)
 382         if settings.has_key("depot-paths"):
 383             paths = ",".join(settings["depot-paths"])
 384             if branchByDepotPath.has_key(paths):
 385                 return [branchByDepotPath[paths], settings]
 386
 387         parent = parent + 1
 388
 389     return ["", settings]
 390
 391 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392     if not silent:
 393         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394                % localRefPrefix)
 395
 396     originPrefix = "origin/p4/"
 397
 398     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399         line = line.strip()
 400         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401             continue
 402
 403         headName = line[len(originPrefix):]
 404         remoteHead = localRefPrefix + headName
 405         originHead = line
 406
 407         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408         if (not original.has_key('depot-paths')
 409             or not original.has_key('change')):
 410             continue
 411
 412         update = False
 413         if not gitBranchExists(remoteHead):
 414             if verbose:
 415                 print "creating %s" % remoteHead
 416             update = True
 417         else:
 418             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419             if settings.has_key('change') > 0:
 420                 if settings['depot-paths'] == original['depot-paths']:
 421                     originP4Change = int(original['change'])
 422                     p4Change = int(settings['change'])
 423                     if originP4Change > p4Change:
 424                         print ("%s (%s) is newer than %s (%s). "
 425                                "Updating p4 branch from origin."
 426                                % (originHead, originP4Change,
 427                                   remoteHead, p4Change))
 428                         update = True
 429                 else:
 430                     print ("Ignoring: %s was imported from %s while "
 431                            "%s was imported from %s"
 432                            % (originHead, ','.join(original['depot-paths']),
 433                               remoteHead, ','.join(settings['depot-paths'])))
 434
 435         if update:
 436             system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438 def originP4BranchesExist():
 439         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441 def p4ChangesForPaths(depotPaths, changeRange):
 442     assert depotPaths
 443     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                         for p in depotPaths]))
 445
 446     changes = {}
 447     for line in output:
 448         changeNum = int(line.split(" ")[1])
 449         changes[changeNum] = True
 450
 451     changelist = changes.keys()
 452     changelist.sort()
 453     return changelist
 454
 455 class Command:
 456     def __init__(self):
 457         self.usage = "usage: %prog [options]"
 458         self.needsGit = True
 459
 460 class P4Debug(Command):
 461     def __init__(self):
 462         Command.__init__(self)
 463         self.options = [
 464             optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                  default=False),
 466             ]
 467         self.description = "A tool to debug the output of p4 -G."
 468         self.needsGit = False
 469         self.verbose = False
 470
 471     def run(self, args):
 472         j = 0
 473         for output in p4CmdList(" ".join(args)):
 474             print 'Element: %d' % j
 475             j += 1
 476             print output
 477         return True
 478
 479 class P4RollBack(Command):
 480     def __init__(self):
 481         Command.__init__(self)
 482         self.options = [
 483             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485         ]
 486         self.description = "A tool to debug the multi-branch import. Don't use :)"
 487         self.verbose = False
 488         self.rollbackLocalBranches = False
 489
 490     def run(self, args):
 491         if len(args) != 1:
 492             return False
 493         maxChange = int(args[0])
 494
 495         if "p4ExitCode" in p4Cmd("changes -m 1"):
 496             die("Problems executing p4");
 497
 498         if self.rollbackLocalBranches:
 499             refPrefix = "refs/heads/"
 500             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501         else:
 502             refPrefix = "refs/remotes/"
 503             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505         for line in lines:
 506             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                 line = line.strip()
 508                 ref = refPrefix + line
 509                 log = extractLogMessageFromGitCommit(ref)
 510                 settings = extractSettingsGitLog(log)
 511
 512                 depotPaths = settings['depot-paths']
 513                 change = settings['change']
 514
 515                 changed = False
 516
 517                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                            for p in depotPaths]))) == 0:
 519                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                     continue
 522
 523                 while change and int(change) > maxChange:
 524                     changed = True
 525                     if self.verbose:
 526                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                     system("git update-ref %s \"%s^\"" % (ref, ref))
 528                     log = extractLogMessageFromGitCommit(ref)
 529                     settings =  extractSettingsGitLog(log)
 530
 531
 532                     depotPaths = settings['depot-paths']
 533                     change = settings['change']
 534
 535                 if changed:
 536                     print "%s rewound to %s" % (ref, change)
 537
 538         return True
 539
 540 class P4Submit(Command):
 541     def __init__(self):
 542         Command.__init__(self)
 543         self.options = [
 544                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                 optparse.make_option("--origin", dest="origin"),
 546                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 547         ]
 548         self.description = "Submit changes from git to the perforce depot."
 549         self.usage += " [name of git branch to submit into perforce depot]"
 550         self.interactive = True
 551         self.origin = ""
 552         self.detectRenames = False
 553         self.verbose = False
 554         self.isWindows = (platform.system() == "Windows")
 555
 556     def check(self):
 557         if len(p4CmdList("opened ...")) > 0:
 558             die("You have files opened with perforce! Close them before starting the sync.")
 559
 560     # replaces everything between 'Description:' and the next P4 submit template field with the
 561     # commit message
 562     def prepareLogMessage(self, template, message):
 563         result = ""
 564
 565         inDescriptionSection = False
 566
 567         for line in template.split("\n"):
 568             if line.startswith("#"):
 569                 result += line + "\n"
 570                 continue
 571
 572             if inDescriptionSection:
 573                 if line.startswith("Files:"):
 574                     inDescriptionSection = False
 575                 else:
 576                     continue
 577             else:
 578                 if line.startswith("Description:"):
 579                     inDescriptionSection = True
 580                     line += "\n"
 581                     for messageLine in message.split("\n"):
 582                         line += "\t" + messageLine + "\n"
 583
 584             result += line + "\n"
 585
 586         return result
 587
 588     def prepareSubmitTemplate(self):
 589         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590         template = ""
 591         inFilesSection = False
 592         for line in p4_read_pipe_lines("change -o"):
 593             if line.endswith("\r\n"):
 594                 line = line[:-2] + "\n"
 595             if inFilesSection:
 596                 if line.startswith("\t"):
 597                     # path starts and ends with a tab
 598                     path = line[1:]
 599                     lastTab = path.rfind("\t")
 600                     if lastTab != -1:
 601                         path = path[:lastTab]
 602                         if not path.startswith(self.depotPath):
 603                             continue
 604                 else:
 605                     inFilesSection = False
 606             else:
 607                 if line.startswith("Files:"):
 608                     inFilesSection = True
 609
 610             template += line
 611
 612         return template
 613
 614     def applyCommit(self, id):
 615         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616
 617         if not self.detectRenames:
 618             # If not explicitly set check the config variable
 619             self.detectRenames = gitConfig("git-p4.detectRenames").lower() == "true"
 620
 621         if self.detectRenames:
 622             diffOpts = "-M"
 623         else:
 624             diffOpts = ""
 625
 626         if gitConfig("git-p4.detectCopies").lower() == "true":
 627             diffOpts += " -C"
 628
 629         if gitConfig("git-p4.detectCopiesHarder").lower() == "true":
 630             diffOpts += " --find-copies-harder"
 631
 632         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 633         filesToAdd = set()
 634         filesToDelete = set()
 635         editedFiles = set()
 636         filesToChangeExecBit = {}
 637         for line in diff:
 638             diff = parseDiffTreeEntry(line)
 639             modifier = diff['status']
 640             path = diff['src']
 641             if modifier == "M":
 642                 p4_system("edit \"%s\"" % path)
 643                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 644                     filesToChangeExecBit[path] = diff['dst_mode']
 645                 editedFiles.add(path)
 646             elif modifier == "A":
 647                 filesToAdd.add(path)
 648                 filesToChangeExecBit[path] = diff['dst_mode']
 649                 if path in filesToDelete:
 650                     filesToDelete.remove(path)
 651             elif modifier == "D":
 652                 filesToDelete.add(path)
 653                 if path in filesToAdd:
 654                     filesToAdd.remove(path)
 655             elif modifier == "C":
 656                 src, dest = diff['src'], diff['dst']
 657                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 658                 if diff['src_sha1'] != diff['dst_sha1']:
 659                     p4_system("edit \"%s\"" % (dest))
 660                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 661                     p4_system("edit \"%s\"" % (dest))
 662                     filesToChangeExecBit[dest] = diff['dst_mode']
 663                 os.unlink(dest)
 664                 editedFiles.add(dest)
 665             elif modifier == "R":
 666                 src, dest = diff['src'], diff['dst']
 667                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 668                 if diff['src_sha1'] != diff['dst_sha1']:
 669                     p4_system("edit \"%s\"" % (dest))
 670                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 671                     p4_system("edit \"%s\"" % (dest))
 672                     filesToChangeExecBit[dest] = diff['dst_mode']
 673                 os.unlink(dest)
 674                 editedFiles.add(dest)
 675                 filesToDelete.add(src)
 676             else:
 677                 die("unknown modifier %s for %s" % (modifier, path))
 678
 679         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 680         patchcmd = diffcmd + " | git apply "
 681         tryPatchCmd = patchcmd + "--check -"
 682         applyPatchCmd = patchcmd + "--check --apply -"
 683
 684         if os.system(tryPatchCmd) != 0:
 685             print "Unfortunately applying the change failed!"
 686             print "What do you want to do?"
 687             response = "x"
 688             while response != "s" and response != "a" and response != "w":
 689                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 690                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 691             if response == "s":
 692                 print "Skipping! Good luck with the next patches..."
 693                 for f in editedFiles:
 694                     p4_system("revert \"%s\"" % f);
 695                 for f in filesToAdd:
 696                     system("rm %s" %f)
 697                 return
 698             elif response == "a":
 699                 os.system(applyPatchCmd)
 700                 if len(filesToAdd) > 0:
 701                     print "You may also want to call p4 add on the following files:"
 702                     print " ".join(filesToAdd)
 703                 if len(filesToDelete):
 704                     print "The following files should be scheduled for deletion with p4 delete:"
 705                     print " ".join(filesToDelete)
 706                 die("Please resolve and submit the conflict manually and "
 707                     + "continue afterwards with git-p4 submit --continue")
 708             elif response == "w":
 709                 system(diffcmd + " > patch.txt")
 710                 print "Patch saved to patch.txt in %s !" % self.clientPath
 711                 die("Please resolve and submit the conflict manually and "
 712                     "continue afterwards with git-p4 submit --continue")
 713
 714         system(applyPatchCmd)
 715
 716         for f in filesToAdd:
 717             p4_system("add \"%s\"" % f)
 718         for f in filesToDelete:
 719             p4_system("revert \"%s\"" % f)
 720             p4_system("delete \"%s\"" % f)
 721
 722         # Set/clear executable bits
 723         for f in filesToChangeExecBit.keys():
 724             mode = filesToChangeExecBit[f]
 725             setP4ExecBit(f, mode)
 726
 727         logMessage = extractLogMessageFromGitCommit(id)
 728         logMessage = logMessage.strip()
 729
 730         template = self.prepareSubmitTemplate()
 731
 732         if self.interactive:
 733             submitTemplate = self.prepareLogMessage(template, logMessage)
 734             if os.environ.has_key("P4DIFF"):
 735                 del(os.environ["P4DIFF"])
 736             diff = ""
 737             for editedFile in editedFiles:
 738                 diff += p4_read_pipe("diff -du %r" % editedFile)
 739
 740             newdiff = ""
 741             for newFile in filesToAdd:
 742                 newdiff += "==== new file ====\n"
 743                 newdiff += "--- /dev/null\n"
 744                 newdiff += "+++ %s\n" % newFile
 745                 f = open(newFile, "r")
 746                 for line in f.readlines():
 747                     newdiff += "+" + line
 748                 f.close()
 749
 750             separatorLine = "######## everything below this line is just the diff #######\n"
 751
 752             [handle, fileName] = tempfile.mkstemp()
 753             tmpFile = os.fdopen(handle, "w+")
 754             if self.isWindows:
 755                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 756                 separatorLine = separatorLine.replace("\n", "\r\n")
 757                 newdiff = newdiff.replace("\n", "\r\n")
 758             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 759             tmpFile.close()
 760             mtime = os.stat(fileName).st_mtime
 761             if os.environ.has_key("P4EDITOR"):
 762                 editor = os.environ.get("P4EDITOR")
 763             else:
 764                 editor = read_pipe("git var GIT_EDITOR").strip()
 765             system(editor + " " + fileName)
 766
 767             response = "y"
 768             if os.stat(fileName).st_mtime <= mtime:
 769                 response = "x"
 770                 while response != "y" and response != "n":
 771                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 772
 773             if response == "y":
 774                 tmpFile = open(fileName, "rb")
 775                 message = tmpFile.read()
 776                 tmpFile.close()
 777                 submitTemplate = message[:message.index(separatorLine)]
 778                 if self.isWindows:
 779                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 780                 p4_write_pipe("submit -i", submitTemplate)
 781             else:
 782                 for f in editedFiles:
 783                     p4_system("revert \"%s\"" % f);
 784                 for f in filesToAdd:
 785                     p4_system("revert \"%s\"" % f);
 786                     system("rm %s" %f)
 787
 788             os.remove(fileName)
 789         else:
 790             fileName = "submit.txt"
 791             file = open(fileName, "w+")
 792             file.write(self.prepareLogMessage(template, logMessage))
 793             file.close()
 794             print ("Perforce submit template written as %s. "
 795                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 796                    % (fileName, fileName))
 797
 798     def run(self, args):
 799         if len(args) == 0:
 800             self.master = currentGitBranch()
 801             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 802                 die("Detecting current git branch failed!")
 803         elif len(args) == 1:
 804             self.master = args[0]
 805         else:
 806             return False
 807
 808         allowSubmit = gitConfig("git-p4.allowSubmit")
 809         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 810             die("%s is not in git-p4.allowSubmit" % self.master)
 811
 812         [upstream, settings] = findUpstreamBranchPoint()
 813         self.depotPath = settings['depot-paths'][0]
 814         if len(self.origin) == 0:
 815             self.origin = upstream
 816
 817         if self.verbose:
 818             print "Origin branch is " + self.origin
 819
 820         if len(self.depotPath) == 0:
 821             print "Internal error: cannot locate perforce depot path from existing branches"
 822             sys.exit(128)
 823
 824         self.clientPath = p4Where(self.depotPath)
 825
 826         if len(self.clientPath) == 0:
 827             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 828             sys.exit(128)
 829
 830         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 831         self.oldWorkingDirectory = os.getcwd()
 832
 833         chdir(self.clientPath)
 834         print "Synchronizing p4 checkout..."
 835         p4_system("sync ...")
 836
 837         self.check()
 838
 839         commits = []
 840         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 841             commits.append(line.strip())
 842         commits.reverse()
 843
 844         while len(commits) > 0:
 845             commit = commits[0]
 846             commits = commits[1:]
 847             self.applyCommit(commit)
 848             if not self.interactive:
 849                 break
 850
 851         if len(commits) == 0:
 852             print "All changes applied!"
 853             chdir(self.oldWorkingDirectory)
 854
 855             sync = P4Sync()
 856             sync.run([])
 857
 858             rebase = P4Rebase()
 859             rebase.rebase()
 860
 861         return True
 862
 863 class P4Sync(Command):
 864     def __init__(self):
 865         Command.__init__(self)
 866         self.options = [
 867                 optparse.make_option("--branch", dest="branch"),
 868                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 869                 optparse.make_option("--changesfile", dest="changesFile"),
 870                 optparse.make_option("--silent", dest="silent", action="store_true"),
 871                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 872                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 873                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 874                                      help="Import into refs/heads/ , not refs/remotes"),
 875                 optparse.make_option("--max-changes", dest="maxChanges"),
 876                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 877                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 878                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 879                                      help="Only sync files that are included in the Perforce Client Spec")
 880         ]
 881         self.description = """Imports from Perforce into a git repository.\n
 882     example:
 883     //depot/my/project/ -- to import the current head
 884     //depot/my/project/@all -- to import everything
 885     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 886
 887     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 888
 889         self.usage += " //depot/path[@revRange]"
 890         self.silent = False
 891         self.createdBranches = set()
 892         self.committedChanges = set()
 893         self.branch = ""
 894         self.detectBranches = False
 895         self.detectLabels = False
 896         self.changesFile = ""
 897         self.syncWithOrigin = True
 898         self.verbose = False
 899         self.importIntoRemotes = True
 900         self.maxChanges = ""
 901         self.isWindows = (platform.system() == "Windows")
 902         self.keepRepoPath = False
 903         self.depotPaths = None
 904         self.p4BranchesInGit = []
 905         self.cloneExclude = []
 906         self.useClientSpec = False
 907         self.clientSpecDirs = []
 908
 909         if gitConfig("git-p4.syncFromOrigin") == "false":
 910             self.syncWithOrigin = False
 911
 912     def extractFilesFromCommit(self, commit):
 913         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 914                              for path in self.cloneExclude]
 915         files = []
 916         fnum = 0
 917         while commit.has_key("depotFile%s" % fnum):
 918             path =  commit["depotFile%s" % fnum]
 919
 920             if [p for p in self.cloneExclude
 921                 if path.startswith (p)]:
 922                 found = False
 923             else:
 924                 found = [p for p in self.depotPaths
 925                          if path.startswith (p)]
 926             if not found:
 927                 fnum = fnum + 1
 928                 continue
 929
 930             file = {}
 931             file["path"] = path
 932             file["rev"] = commit["rev%s" % fnum]
 933             file["action"] = commit["action%s" % fnum]
 934             file["type"] = commit["type%s" % fnum]
 935             files.append(file)
 936             fnum = fnum + 1
 937         return files
 938
 939     def stripRepoPath(self, path, prefixes):
 940         if self.keepRepoPath:
 941             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 942
 943         for p in prefixes:
 944             if path.startswith(p):
 945                 path = path[len(p):]
 946
 947         return path
 948
 949     def splitFilesIntoBranches(self, commit):
 950         branches = {}
 951         fnum = 0
 952         while commit.has_key("depotFile%s" % fnum):
 953             path =  commit["depotFile%s" % fnum]
 954             found = [p for p in self.depotPaths
 955                      if path.startswith (p)]
 956             if not found:
 957                 fnum = fnum + 1
 958                 continue
 959
 960             file = {}
 961             file["path"] = path
 962             file["rev"] = commit["rev%s" % fnum]
 963             file["action"] = commit["action%s" % fnum]
 964             file["type"] = commit["type%s" % fnum]
 965             fnum = fnum + 1
 966
 967             relPath = self.stripRepoPath(path, self.depotPaths)
 968
 969             for branch in self.knownBranches.keys():
 970
 971                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 972                 if relPath.startswith(branch + "/"):
 973                     if branch not in branches:
 974                         branches[branch] = []
 975                     branches[branch].append(file)
 976                     break
 977
 978         return branches
 979
 980     # output one file from the P4 stream
 981     # - helper for streamP4Files
 982
 983     def streamOneP4File(self, file, contents):
 984         if file["type"] == "apple":
 985             print "\nfile %s is a strange apple file that forks. Ignoring" % \
 986                 file['depotFile']
 987             return
 988
 989         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 990         if verbose:
 991             sys.stderr.write("%s\n" % relPath)
 992
 993         mode = "644"
 994         if isP4Exec(file["type"]):
 995             mode = "755"
 996         elif file["type"] == "symlink":
 997             mode = "120000"
 998             # p4 print on a symlink contains "target\n", so strip it off
 999             data = ''.join(contents)
1000             contents = [data[:-1]]
1001
1002         if self.isWindows and file["type"].endswith("text"):
1003             mangled = []
1004             for data in contents:
1005                 data = data.replace("\r\n", "\n")
1006                 mangled.append(data)
1007             contents = mangled
1008
1009         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1010             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1011         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1012             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1013
1014         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1015
1016         # total length...
1017         length = 0
1018         for d in contents:
1019             length = length + len(d)
1020
1021         self.gitStream.write("data %d\n" % length)
1022         for d in contents:
1023             self.gitStream.write(d)
1024         self.gitStream.write("\n")
1025
1026     def streamOneP4Deletion(self, file):
1027         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1028         if verbose:
1029             sys.stderr.write("delete %s\n" % relPath)
1030         self.gitStream.write("D %s\n" % relPath)
1031
1032     # handle another chunk of streaming data
1033     def streamP4FilesCb(self, marshalled):
1034
1035         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1036             # start of a new file - output the old one first
1037             self.streamOneP4File(self.stream_file, self.stream_contents)
1038             self.stream_file = {}
1039             self.stream_contents = []
1040             self.stream_have_file_info = False
1041
1042         # pick up the new file information... for the
1043         # 'data' field we need to append to our array
1044         for k in marshalled.keys():
1045             if k == 'data':
1046                 self.stream_contents.append(marshalled['data'])
1047             else:
1048                 self.stream_file[k] = marshalled[k]
1049
1050         self.stream_have_file_info = True
1051
1052     # Stream directly from "p4 files" into "git fast-import"
1053     def streamP4Files(self, files):
1054         filesForCommit = []
1055         filesToRead = []
1056         filesToDelete = []
1057
1058         for f in files:
1059             includeFile = True
1060             for val in self.clientSpecDirs:
1061                 if f['path'].startswith(val[0]):
1062                     if val[1] <= 0:
1063                         includeFile = False
1064                     break
1065
1066             if includeFile:
1067                 filesForCommit.append(f)
1068                 if f['action'] not in ('delete', 'move/delete', 'purge'):
1069                     filesToRead.append(f)
1070                 else:
1071                     filesToDelete.append(f)
1072
1073         # deleted files...
1074         for f in filesToDelete:
1075             self.streamOneP4Deletion(f)
1076
1077         if len(filesToRead) > 0:
1078             self.stream_file = {}
1079             self.stream_contents = []
1080             self.stream_have_file_info = False
1081
1082             # curry self argument
1083             def streamP4FilesCbSelf(entry):
1084                 self.streamP4FilesCb(entry)
1085
1086             p4CmdList("-x - print",
1087                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1088                                                   for f in filesToRead]),
1089                 cb=streamP4FilesCbSelf)
1090
1091             # do the last chunk
1092             if self.stream_file.has_key('depotFile'):
1093                 self.streamOneP4File(self.stream_file, self.stream_contents)
1094
1095     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1096         epoch = details["time"]
1097         author = details["user"]
1098         self.branchPrefixes = branchPrefixes
1099
1100         if self.verbose:
1101             print "commit into %s" % branch
1102
1103         # start with reading files; if that fails, we should not
1104         # create a commit.
1105         new_files = []
1106         for f in files:
1107             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1108                 new_files.append (f)
1109             else:
1110                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1111
1112         self.gitStream.write("commit %s\n" % branch)
1113 #        gitStream.write("mark :%s\n" % details["change"])
1114         self.committedChanges.add(int(details["change"]))
1115         committer = ""
1116         if author not in self.users:
1117             self.getUserMapFromPerforceServer()
1118         if author in self.users:
1119             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1120         else:
1121             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1122
1123         self.gitStream.write("committer %s\n" % committer)
1124
1125         self.gitStream.write("data <<EOT\n")
1126         self.gitStream.write(details["desc"])
1127         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1128                              % (','.join (branchPrefixes), details["change"]))
1129         if len(details['options']) > 0:
1130             self.gitStream.write(": options = %s" % details['options'])
1131         self.gitStream.write("]\nEOT\n\n")
1132
1133         if len(parent) > 0:
1134             if self.verbose:
1135                 print "parent %s" % parent
1136             self.gitStream.write("from %s\n" % parent)
1137
1138         self.streamP4Files(new_files)
1139         self.gitStream.write("\n")
1140
1141         change = int(details["change"])
1142
1143         if self.labels.has_key(change):
1144             label = self.labels[change]
1145             labelDetails = label[0]
1146             labelRevisions = label[1]
1147             if self.verbose:
1148                 print "Change %s is labelled %s" % (change, labelDetails)
1149
1150             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1151                                                     for p in branchPrefixes]))
1152
1153             if len(files) == len(labelRevisions):
1154
1155                 cleanedFiles = {}
1156                 for info in files:
1157                     if info["action"] in ("delete", "purge"):
1158                         continue
1159                     cleanedFiles[info["depotFile"]] = info["rev"]
1160
1161                 if cleanedFiles == labelRevisions:
1162                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1163                     self.gitStream.write("from %s\n" % branch)
1164
1165                     owner = labelDetails["Owner"]
1166                     tagger = ""
1167                     if author in self.users:
1168                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1169                     else:
1170                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1171                     self.gitStream.write("tagger %s\n" % tagger)
1172                     self.gitStream.write("data <<EOT\n")
1173                     self.gitStream.write(labelDetails["Description"])
1174                     self.gitStream.write("EOT\n\n")
1175
1176                 else:
1177                     if not self.silent:
1178                         print ("Tag %s does not match with change %s: files do not match."
1179                                % (labelDetails["label"], change))
1180
1181             else:
1182                 if not self.silent:
1183                     print ("Tag %s does not match with change %s: file count is different."
1184                            % (labelDetails["label"], change))
1185
1186     def getUserCacheFilename(self):
1187         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1188         return home + "/.gitp4-usercache.txt"
1189
1190     def getUserMapFromPerforceServer(self):
1191         if self.userMapFromPerforceServer:
1192             return
1193         self.users = {}
1194
1195         for output in p4CmdList("users"):
1196             if not output.has_key("User"):
1197                 continue
1198             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1199
1200
1201         s = ''
1202         for (key, val) in self.users.items():
1203             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1204
1205         open(self.getUserCacheFilename(), "wb").write(s)
1206         self.userMapFromPerforceServer = True
1207
1208     def loadUserMapFromCache(self):
1209         self.users = {}
1210         self.userMapFromPerforceServer = False
1211         try:
1212             cache = open(self.getUserCacheFilename(), "rb")
1213             lines = cache.readlines()
1214             cache.close()
1215             for line in lines:
1216                 entry = line.strip().split("\t")
1217                 self.users[entry[0]] = entry[1]
1218         except IOError:
1219             self.getUserMapFromPerforceServer()
1220
1221     def getLabels(self):
1222         self.labels = {}
1223
1224         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1225         if len(l) > 0 and not self.silent:
1226             print "Finding files belonging to labels in %s" % `self.depotPaths`
1227
1228         for output in l:
1229             label = output["label"]
1230             revisions = {}
1231             newestChange = 0
1232             if self.verbose:
1233                 print "Querying files for label %s" % label
1234             for file in p4CmdList("files "
1235                                   +  ' '.join (["%s...@%s" % (p, label)
1236                                                 for p in self.depotPaths])):
1237                 revisions[file["depotFile"]] = file["rev"]
1238                 change = int(file["change"])
1239                 if change > newestChange:
1240                     newestChange = change
1241
1242             self.labels[newestChange] = [output, revisions]
1243
1244         if self.verbose:
1245             print "Label changes: %s" % self.labels.keys()
1246
1247     def guessProjectName(self):
1248         for p in self.depotPaths:
1249             if p.endswith("/"):
1250                 p = p[:-1]
1251             p = p[p.strip().rfind("/") + 1:]
1252             if not p.endswith("/"):
1253                p += "/"
1254             return p
1255
1256     def getBranchMapping(self):
1257         lostAndFoundBranches = set()
1258
1259         for info in p4CmdList("branches"):
1260             details = p4Cmd("branch -o %s" % info["branch"])
1261             viewIdx = 0
1262             while details.has_key("View%s" % viewIdx):
1263                 paths = details["View%s" % viewIdx].split(" ")
1264                 viewIdx = viewIdx + 1
1265                 # require standard //depot/foo/... //depot/bar/... mapping
1266                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1267                     continue
1268                 source = paths[0]
1269                 destination = paths[1]
1270                 ## HACK
1271                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1272                     source = source[len(self.depotPaths[0]):-4]
1273                     destination = destination[len(self.depotPaths[0]):-4]
1274
1275                     if destination in self.knownBranches:
1276                         if not self.silent:
1277                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1278                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1279                         continue
1280
1281                     self.knownBranches[destination] = source
1282
1283                     lostAndFoundBranches.discard(destination)
1284
1285                     if source not in self.knownBranches:
1286                         lostAndFoundBranches.add(source)
1287
1288
1289         for branch in lostAndFoundBranches:
1290             self.knownBranches[branch] = branch
1291
1292     def getBranchMappingFromGitBranches(self):
1293         branches = p4BranchesInGit(self.importIntoRemotes)
1294         for branch in branches.keys():
1295             if branch == "master":
1296                 branch = "main"
1297             else:
1298                 branch = branch[len(self.projectName):]
1299             self.knownBranches[branch] = branch
1300
1301     def listExistingP4GitBranches(self):
1302         # branches holds mapping from name to commit
1303         branches = p4BranchesInGit(self.importIntoRemotes)
1304         self.p4BranchesInGit = branches.keys()
1305         for branch in branches.keys():
1306             self.initialParents[self.refPrefix + branch] = branches[branch]
1307
1308     def updateOptionDict(self, d):
1309         option_keys = {}
1310         if self.keepRepoPath:
1311             option_keys['keepRepoPath'] = 1
1312
1313         d["options"] = ' '.join(sorted(option_keys.keys()))
1314
1315     def readOptions(self, d):
1316         self.keepRepoPath = (d.has_key('options')
1317                              and ('keepRepoPath' in d['options']))
1318
1319     def gitRefForBranch(self, branch):
1320         if branch == "main":
1321             return self.refPrefix + "master"
1322
1323         if len(branch) <= 0:
1324             return branch
1325
1326         return self.refPrefix + self.projectName + branch
1327
1328     def gitCommitByP4Change(self, ref, change):
1329         if self.verbose:
1330             print "looking in ref " + ref + " for change %s using bisect..." % change
1331
1332         earliestCommit = ""
1333         latestCommit = parseRevision(ref)
1334
1335         while True:
1336             if self.verbose:
1337                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1338             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1339             if len(next) == 0:
1340                 if self.verbose:
1341                     print "argh"
1342                 return ""
1343             log = extractLogMessageFromGitCommit(next)
1344             settings = extractSettingsGitLog(log)
1345             currentChange = int(settings['change'])
1346             if self.verbose:
1347                 print "current change %s" % currentChange
1348
1349             if currentChange == change:
1350                 if self.verbose:
1351                     print "found %s" % next
1352                 return next
1353
1354             if currentChange < change:
1355                 earliestCommit = "^%s" % next
1356             else:
1357                 latestCommit = "%s" % next
1358
1359         return ""
1360
1361     def importNewBranch(self, branch, maxChange):
1362         # make fast-import flush all changes to disk and update the refs using the checkpoint
1363         # command so that we can try to find the branch parent in the git history
1364         self.gitStream.write("checkpoint\n\n");
1365         self.gitStream.flush();
1366         branchPrefix = self.depotPaths[0] + branch + "/"
1367         range = "@1,%s" % maxChange
1368         #print "prefix" + branchPrefix
1369         changes = p4ChangesForPaths([branchPrefix], range)
1370         if len(changes) <= 0:
1371             return False
1372         firstChange = changes[0]
1373         #print "first change in branch: %s" % firstChange
1374         sourceBranch = self.knownBranches[branch]
1375         sourceDepotPath = self.depotPaths[0] + sourceBranch
1376         sourceRef = self.gitRefForBranch(sourceBranch)
1377         #print "source " + sourceBranch
1378
1379         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1380         #print "branch parent: %s" % branchParentChange
1381         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1382         if len(gitParent) > 0:
1383             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1384             #print "parent git commit: %s" % gitParent
1385
1386         self.importChanges(changes)
1387         return True
1388
1389     def importChanges(self, changes):
1390         cnt = 1
1391         for change in changes:
1392             description = p4Cmd("describe %s" % change)
1393             self.updateOptionDict(description)
1394
1395             if not self.silent:
1396                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1397                 sys.stdout.flush()
1398             cnt = cnt + 1
1399
1400             try:
1401                 if self.detectBranches:
1402                     branches = self.splitFilesIntoBranches(description)
1403                     for branch in branches.keys():
1404                         ## HACK  --hwn
1405                         branchPrefix = self.depotPaths[0] + branch + "/"
1406
1407                         parent = ""
1408
1409                         filesForCommit = branches[branch]
1410
1411                         if self.verbose:
1412                             print "branch is %s" % branch
1413
1414                         self.updatedBranches.add(branch)
1415
1416                         if branch not in self.createdBranches:
1417                             self.createdBranches.add(branch)
1418                             parent = self.knownBranches[branch]
1419                             if parent == branch:
1420                                 parent = ""
1421                             else:
1422                                 fullBranch = self.projectName + branch
1423                                 if fullBranch not in self.p4BranchesInGit:
1424                                     if not self.silent:
1425                                         print("\n    Importing new branch %s" % fullBranch);
1426                                     if self.importNewBranch(branch, change - 1):
1427                                         parent = ""
1428                                         self.p4BranchesInGit.append(fullBranch)
1429                                     if not self.silent:
1430                                         print("\n    Resuming with change %s" % change);
1431
1432                                 if self.verbose:
1433                                     print "parent determined through known branches: %s" % parent
1434
1435                         branch = self.gitRefForBranch(branch)
1436                         parent = self.gitRefForBranch(parent)
1437
1438                         if self.verbose:
1439                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1440
1441                         if len(parent) == 0 and branch in self.initialParents:
1442                             parent = self.initialParents[branch]
1443                             del self.initialParents[branch]
1444
1445                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1446                 else:
1447                     files = self.extractFilesFromCommit(description)
1448                     self.commit(description, files, self.branch, self.depotPaths,
1449                                 self.initialParent)
1450                     self.initialParent = ""
1451             except IOError:
1452                 print self.gitError.read()
1453                 sys.exit(1)
1454
1455     def importHeadRevision(self, revision):
1456         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1457
1458         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1459         details["desc"] = ("Initial import of %s from the state at revision %s"
1460                            % (' '.join(self.depotPaths), revision))
1461         details["change"] = revision
1462         newestRevision = 0
1463
1464         fileCnt = 0
1465         for info in p4CmdList("files "
1466                               +  ' '.join(["%s...%s"
1467                                            % (p, revision)
1468                                            for p in self.depotPaths])):
1469
1470             if info['code'] == 'error':
1471                 sys.stderr.write("p4 returned an error: %s\n"
1472                                  % info['data'])
1473                 sys.exit(1)
1474
1475
1476             change = int(info["change"])
1477             if change > newestRevision:
1478                 newestRevision = change
1479
1480             if info["action"] in ("delete", "purge"):
1481                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1482                 #fileCnt = fileCnt + 1
1483                 continue
1484
1485             for prop in ["depotFile", "rev", "action", "type" ]:
1486                 details["%s%s" % (prop, fileCnt)] = info[prop]
1487
1488             fileCnt = fileCnt + 1
1489
1490         details["change"] = newestRevision
1491         self.updateOptionDict(details)
1492         try:
1493             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1494         except IOError:
1495             print "IO error with git fast-import. Is your git version recent enough?"
1496             print self.gitError.read()
1497
1498
1499     def getClientSpec(self):
1500         specList = p4CmdList( "client -o" )
1501         temp = {}
1502         for entry in specList:
1503             for k,v in entry.iteritems():
1504                 if k.startswith("View"):
1505                     if v.startswith('"'):
1506                         start = 1
1507                     else:
1508                         start = 0
1509                     index = v.find("...")
1510                     v = v[start:index]
1511                     if v.startswith("-"):
1512                         v = v[1:]
1513                         temp[v] = -len(v)
1514                     else:
1515                         temp[v] = len(v)
1516         self.clientSpecDirs = temp.items()
1517         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1518
1519     def run(self, args):
1520         self.depotPaths = []
1521         self.changeRange = ""
1522         self.initialParent = ""
1523         self.previousDepotPaths = []
1524
1525         # map from branch depot path to parent branch
1526         self.knownBranches = {}
1527         self.initialParents = {}
1528         self.hasOrigin = originP4BranchesExist()
1529         if not self.syncWithOrigin:
1530             self.hasOrigin = False
1531
1532         if self.importIntoRemotes:
1533             self.refPrefix = "refs/remotes/p4/"
1534         else:
1535             self.refPrefix = "refs/heads/p4/"
1536
1537         if self.syncWithOrigin and self.hasOrigin:
1538             if not self.silent:
1539                 print "Syncing with origin first by calling git fetch origin"
1540             system("git fetch origin")
1541
1542         if len(self.branch) == 0:
1543             self.branch = self.refPrefix + "master"
1544             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1545                 system("git update-ref %s refs/heads/p4" % self.branch)
1546                 system("git branch -D p4");
1547             # create it /after/ importing, when master exists
1548             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1549                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1550
1551         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1552             self.getClientSpec()
1553
1554         # TODO: should always look at previous commits,
1555         # merge with previous imports, if possible.
1556         if args == []:
1557             if self.hasOrigin:
1558                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1559             self.listExistingP4GitBranches()
1560
1561             if len(self.p4BranchesInGit) > 1:
1562                 if not self.silent:
1563                     print "Importing from/into multiple branches"
1564                 self.detectBranches = True
1565
1566             if self.verbose:
1567                 print "branches: %s" % self.p4BranchesInGit
1568
1569             p4Change = 0
1570             for branch in self.p4BranchesInGit:
1571                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1572
1573                 settings = extractSettingsGitLog(logMsg)
1574
1575                 self.readOptions(settings)
1576                 if (settings.has_key('depot-paths')
1577                     and settings.has_key ('change')):
1578                     change = int(settings['change']) + 1
1579                     p4Change = max(p4Change, change)
1580
1581                     depotPaths = sorted(settings['depot-paths'])
1582                     if self.previousDepotPaths == []:
1583                         self.previousDepotPaths = depotPaths
1584                     else:
1585                         paths = []
1586                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1587                             for i in range(0, min(len(cur), len(prev))):
1588                                 if cur[i] <> prev[i]:
1589                                     i = i - 1
1590                                     break
1591
1592                             paths.append (cur[:i + 1])
1593
1594                         self.previousDepotPaths = paths
1595
1596             if p4Change > 0:
1597                 self.depotPaths = sorted(self.previousDepotPaths)
1598                 self.changeRange = "@%s,#head" % p4Change
1599                 if not self.detectBranches:
1600                     self.initialParent = parseRevision(self.branch)
1601                 if not self.silent and not self.detectBranches:
1602                     print "Performing incremental import into %s git branch" % self.branch
1603
1604         if not self.branch.startswith("refs/"):
1605             self.branch = "refs/heads/" + self.branch
1606
1607         if len(args) == 0 and self.depotPaths:
1608             if not self.silent:
1609                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1610         else:
1611             if self.depotPaths and self.depotPaths != args:
1612                 print ("previous import used depot path %s and now %s was specified. "
1613                        "This doesn't work!" % (' '.join (self.depotPaths),
1614                                                ' '.join (args)))
1615                 sys.exit(1)
1616
1617             self.depotPaths = sorted(args)
1618
1619         revision = ""
1620         self.users = {}
1621
1622         newPaths = []
1623         for p in self.depotPaths:
1624             if p.find("@") != -1:
1625                 atIdx = p.index("@")
1626                 self.changeRange = p[atIdx:]
1627                 if self.changeRange == "@all":
1628                     self.changeRange = ""
1629                 elif ',' not in self.changeRange:
1630                     revision = self.changeRange
1631                     self.changeRange = ""
1632                 p = p[:atIdx]
1633             elif p.find("#") != -1:
1634                 hashIdx = p.index("#")
1635                 revision = p[hashIdx:]
1636                 p = p[:hashIdx]
1637             elif self.previousDepotPaths == []:
1638                 revision = "#head"
1639
1640             p = re.sub ("\.\.\.$", "", p)
1641             if not p.endswith("/"):
1642                 p += "/"
1643
1644             newPaths.append(p)
1645
1646         self.depotPaths = newPaths
1647
1648
1649         self.loadUserMapFromCache()
1650         self.labels = {}
1651         if self.detectLabels:
1652             self.getLabels();
1653
1654         if self.detectBranches:
1655             ## FIXME - what's a P4 projectName ?
1656             self.projectName = self.guessProjectName()
1657
1658             if self.hasOrigin:
1659                 self.getBranchMappingFromGitBranches()
1660             else:
1661                 self.getBranchMapping()
1662             if self.verbose:
1663                 print "p4-git branches: %s" % self.p4BranchesInGit
1664                 print "initial parents: %s" % self.initialParents
1665             for b in self.p4BranchesInGit:
1666                 if b != "master":
1667
1668                     ## FIXME
1669                     b = b[len(self.projectName):]
1670                 self.createdBranches.add(b)
1671
1672         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1673
1674         importProcess = subprocess.Popen(["git", "fast-import"],
1675                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1676                                          stderr=subprocess.PIPE);
1677         self.gitOutput = importProcess.stdout
1678         self.gitStream = importProcess.stdin
1679         self.gitError = importProcess.stderr
1680
1681         if revision:
1682             self.importHeadRevision(revision)
1683         else:
1684             changes = []
1685
1686             if len(self.changesFile) > 0:
1687                 output = open(self.changesFile).readlines()
1688                 changeSet = set()
1689                 for line in output:
1690                     changeSet.add(int(line))
1691
1692                 for change in changeSet:
1693                     changes.append(change)
1694
1695                 changes.sort()
1696             else:
1697                 if self.verbose:
1698                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1699                                                               self.changeRange)
1700                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1701
1702                 if len(self.maxChanges) > 0:
1703                     changes = changes[:min(int(self.maxChanges), len(changes))]
1704
1705             if len(changes) == 0:
1706                 if not self.silent:
1707                     print "No changes to import!"
1708                 return True
1709
1710             if not self.silent and not self.detectBranches:
1711                 print "Import destination: %s" % self.branch
1712
1713             self.updatedBranches = set()
1714
1715             self.importChanges(changes)
1716
1717             if not self.silent:
1718                 print ""
1719                 if len(self.updatedBranches) > 0:
1720                     sys.stdout.write("Updated branches: ")
1721                     for b in self.updatedBranches:
1722                         sys.stdout.write("%s " % b)
1723                     sys.stdout.write("\n")
1724
1725         self.gitStream.close()
1726         if importProcess.wait() != 0:
1727             die("fast-import failed: %s" % self.gitError.read())
1728         self.gitOutput.close()
1729         self.gitError.close()
1730
1731         return True
1732
1733 class P4Rebase(Command):
1734     def __init__(self):
1735         Command.__init__(self)
1736         self.options = [ ]
1737         self.description = ("Fetches the latest revision from perforce and "
1738                             + "rebases the current work (branch) against it")
1739         self.verbose = False
1740
1741     def run(self, args):
1742         sync = P4Sync()
1743         sync.run([])
1744
1745         return self.rebase()
1746
1747     def rebase(self):
1748         if os.system("git update-index --refresh") != 0:
1749             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1750         if len(read_pipe("git diff-index HEAD --")) > 0:
1751             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1752
1753         [upstream, settings] = findUpstreamBranchPoint()
1754         if len(upstream) == 0:
1755             die("Cannot find upstream branchpoint for rebase")
1756
1757         # the branchpoint may be p4/foo~3, so strip off the parent
1758         upstream = re.sub("~[0-9]+$", "", upstream)
1759
1760         print "Rebasing the current branch onto %s" % upstream
1761         oldHead = read_pipe("git rev-parse HEAD").strip()
1762         system("git rebase %s" % upstream)
1763         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1764         return True
1765
1766 class P4Clone(P4Sync):
1767     def __init__(self):
1768         P4Sync.__init__(self)
1769         self.description = "Creates a new git repository and imports from Perforce into it"
1770         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1771         self.options += [
1772             optparse.make_option("--destination", dest="cloneDestination",
1773                                  action='store', default=None,
1774                                  help="where to leave result of the clone"),
1775             optparse.make_option("-/", dest="cloneExclude",
1776                                  action="append", type="string",
1777                                  help="exclude depot path")
1778         ]
1779         self.cloneDestination = None
1780         self.needsGit = False
1781
1782     # This is required for the "append" cloneExclude action
1783     def ensure_value(self, attr, value):
1784         if not hasattr(self, attr) or getattr(self, attr) is None:
1785             setattr(self, attr, value)
1786         return getattr(self, attr)
1787
1788     def defaultDestination(self, args):
1789         ## TODO: use common prefix of args?
1790         depotPath = args[0]
1791         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1792         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1793         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1794         depotDir = re.sub(r"/$", "", depotDir)
1795         return os.path.split(depotDir)[1]
1796
1797     def run(self, args):
1798         if len(args) < 1:
1799             return False
1800
1801         if self.keepRepoPath and not self.cloneDestination:
1802             sys.stderr.write("Must specify destination for --keep-path\n")
1803             sys.exit(1)
1804
1805         depotPaths = args
1806
1807         if not self.cloneDestination and len(depotPaths) > 1:
1808             self.cloneDestination = depotPaths[-1]
1809             depotPaths = depotPaths[:-1]
1810
1811         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1812         for p in depotPaths:
1813             if not p.startswith("//"):
1814                 return False
1815
1816         if not self.cloneDestination:
1817             self.cloneDestination = self.defaultDestination(args)
1818
1819         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1820         if not os.path.exists(self.cloneDestination):
1821             os.makedirs(self.cloneDestination)
1822         chdir(self.cloneDestination)
1823         system("git init")
1824         self.gitdir = os.getcwd() + "/.git"
1825         if not P4Sync.run(self, depotPaths):
1826             return False
1827         if self.branch != "master":
1828             if self.importIntoRemotes:
1829                 masterbranch = "refs/remotes/p4/master"
1830             else:
1831                 masterbranch = "refs/heads/p4/master"
1832             if gitBranchExists(masterbranch):
1833                 system("git branch master %s" % masterbranch)
1834                 system("git checkout -f")
1835             else:
1836                 print "Could not detect main branch. No checkout/master branch created."
1837
1838         return True
1839
1840 class P4Branches(Command):
1841     def __init__(self):
1842         Command.__init__(self)
1843         self.options = [ ]
1844         self.description = ("Shows the git branches that hold imports and their "
1845                             + "corresponding perforce depot paths")
1846         self.verbose = False
1847
1848     def run(self, args):
1849         if originP4BranchesExist():
1850             createOrUpdateBranchesFromOrigin()
1851
1852         cmdline = "git rev-parse --symbolic "
1853         cmdline += " --remotes"
1854
1855         for line in read_pipe_lines(cmdline):
1856             line = line.strip()
1857
1858             if not line.startswith('p4/') or line == "p4/HEAD":
1859                 continue
1860             branch = line
1861
1862             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1863             settings = extractSettingsGitLog(log)
1864
1865             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1866         return True
1867
1868 class HelpFormatter(optparse.IndentedHelpFormatter):
1869     def __init__(self):
1870         optparse.IndentedHelpFormatter.__init__(self)
1871
1872     def format_description(self, description):
1873         if description:
1874             return description + "\n"
1875         else:
1876             return ""
1877
1878 def printUsage(commands):
1879     print "usage: %s <command> [options]" % sys.argv[0]
1880     print ""
1881     print "valid commands: %s" % ", ".join(commands)
1882     print ""
1883     print "Try %s <command> --help for command specific help." % sys.argv[0]
1884     print ""
1885
1886 commands = {
1887     "debug" : P4Debug,
1888     "submit" : P4Submit,
1889     "commit" : P4Submit,
1890     "sync" : P4Sync,
1891     "rebase" : P4Rebase,
1892     "clone" : P4Clone,
1893     "rollback" : P4RollBack,
1894     "branches" : P4Branches
1895 }
1896
1897
1898 def main():
1899     if len(sys.argv[1:]) == 0:
1900         printUsage(commands.keys())
1901         sys.exit(2)
1902
1903     cmd = ""
1904     cmdName = sys.argv[1]
1905     try:
1906         klass = commands[cmdName]
1907         cmd = klass()
1908     except KeyError:
1909         print "unknown command %s" % cmdName
1910         print ""
1911         printUsage(commands.keys())
1912         sys.exit(2)
1913
1914     options = cmd.options
1915     cmd.gitdir = os.environ.get("GIT_DIR", None)
1916
1917     args = sys.argv[2:]
1918
1919     if len(options) > 0:
1920         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1921
1922         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1923                                        options,
1924                                        description = cmd.description,
1925                                        formatter = HelpFormatter())
1926
1927         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1928     global verbose
1929     verbose = cmd.verbose
1930     if cmd.needsGit:
1931         if cmd.gitdir == None:
1932             cmd.gitdir = os.path.abspath(".git")
1933             if not isValidGitDir(cmd.gitdir):
1934                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1935                 if os.path.exists(cmd.gitdir):
1936                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1937                     if len(cdup) > 0:
1938                         chdir(cdup);
1939
1940         if not isValidGitDir(cmd.gitdir):
1941             if isValidGitDir(cmd.gitdir + "/.git"):
1942                 cmd.gitdir += "/.git"
1943             else:
1944                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1945
1946         os.environ["GIT_DIR"] = cmd.gitdir
1947
1948     if not cmd.run(args):
1949         parser.print_help()
1950
1951
1952 if __name__ == '__main__':
1953     main()