contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key):
 337     if not _gitConfig.has_key(key):
 338         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339     return _gitConfig[key]
 340
 341 def p4BranchesInGit(branchesAreInRemotes = True):
 342     branches = {}
 343
 344     cmdline = "git rev-parse --symbolic "
 345     if branchesAreInRemotes:
 346         cmdline += " --remotes"
 347     else:
 348         cmdline += " --branches"
 349
 350     for line in read_pipe_lines(cmdline):
 351         line = line.strip()
 352
 353         ## only import to p4/
 354         if not line.startswith('p4/') or line == "p4/HEAD":
 355             continue
 356         branch = line
 357
 358         # strip off p4
 359         branch = re.sub ("^p4/", "", line)
 360
 361         branches[branch] = parseRevision(line)
 362     return branches
 363
 364 def findUpstreamBranchPoint(head = "HEAD"):
 365     branches = p4BranchesInGit()
 366     # map from depot-path to branch name
 367     branchByDepotPath = {}
 368     for branch in branches.keys():
 369         tip = branches[branch]
 370         log = extractLogMessageFromGitCommit(tip)
 371         settings = extractSettingsGitLog(log)
 372         if settings.has_key("depot-paths"):
 373             paths = ",".join(settings["depot-paths"])
 374             branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376     settings = None
 377     parent = 0
 378     while parent < 65535:
 379         commit = head + "~%s" % parent
 380         log = extractLogMessageFromGitCommit(commit)
 381         settings = extractSettingsGitLog(log)
 382         if settings.has_key("depot-paths"):
 383             paths = ",".join(settings["depot-paths"])
 384             if branchByDepotPath.has_key(paths):
 385                 return [branchByDepotPath[paths], settings]
 386
 387         parent = parent + 1
 388
 389     return ["", settings]
 390
 391 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392     if not silent:
 393         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394                % localRefPrefix)
 395
 396     originPrefix = "origin/p4/"
 397
 398     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399         line = line.strip()
 400         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401             continue
 402
 403         headName = line[len(originPrefix):]
 404         remoteHead = localRefPrefix + headName
 405         originHead = line
 406
 407         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408         if (not original.has_key('depot-paths')
 409             or not original.has_key('change')):
 410             continue
 411
 412         update = False
 413         if not gitBranchExists(remoteHead):
 414             if verbose:
 415                 print "creating %s" % remoteHead
 416             update = True
 417         else:
 418             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419             if settings.has_key('change') > 0:
 420                 if settings['depot-paths'] == original['depot-paths']:
 421                     originP4Change = int(original['change'])
 422                     p4Change = int(settings['change'])
 423                     if originP4Change > p4Change:
 424                         print ("%s (%s) is newer than %s (%s). "
 425                                "Updating p4 branch from origin."
 426                                % (originHead, originP4Change,
 427                                   remoteHead, p4Change))
 428                         update = True
 429                 else:
 430                     print ("Ignoring: %s was imported from %s while "
 431                            "%s was imported from %s"
 432                            % (originHead, ','.join(original['depot-paths']),
 433                               remoteHead, ','.join(settings['depot-paths'])))
 434
 435         if update:
 436             system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438 def originP4BranchesExist():
 439         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441 def p4ChangesForPaths(depotPaths, changeRange):
 442     assert depotPaths
 443     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                         for p in depotPaths]))
 445
 446     changes = {}
 447     for line in output:
 448         changeNum = int(line.split(" ")[1])
 449         changes[changeNum] = True
 450
 451     changelist = changes.keys()
 452     changelist.sort()
 453     return changelist
 454
 455 class Command:
 456     def __init__(self):
 457         self.usage = "usage: %prog [options]"
 458         self.needsGit = True
 459
 460 class P4Debug(Command):
 461     def __init__(self):
 462         Command.__init__(self)
 463         self.options = [
 464             optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                  default=False),
 466             ]
 467         self.description = "A tool to debug the output of p4 -G."
 468         self.needsGit = False
 469         self.verbose = False
 470
 471     def run(self, args):
 472         j = 0
 473         for output in p4CmdList(" ".join(args)):
 474             print 'Element: %d' % j
 475             j += 1
 476             print output
 477         return True
 478
 479 class P4RollBack(Command):
 480     def __init__(self):
 481         Command.__init__(self)
 482         self.options = [
 483             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485         ]
 486         self.description = "A tool to debug the multi-branch import. Don't use :)"
 487         self.verbose = False
 488         self.rollbackLocalBranches = False
 489
 490     def run(self, args):
 491         if len(args) != 1:
 492             return False
 493         maxChange = int(args[0])
 494
 495         if "p4ExitCode" in p4Cmd("changes -m 1"):
 496             die("Problems executing p4");
 497
 498         if self.rollbackLocalBranches:
 499             refPrefix = "refs/heads/"
 500             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501         else:
 502             refPrefix = "refs/remotes/"
 503             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505         for line in lines:
 506             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                 line = line.strip()
 508                 ref = refPrefix + line
 509                 log = extractLogMessageFromGitCommit(ref)
 510                 settings = extractSettingsGitLog(log)
 511
 512                 depotPaths = settings['depot-paths']
 513                 change = settings['change']
 514
 515                 changed = False
 516
 517                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                            for p in depotPaths]))) == 0:
 519                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                     continue
 522
 523                 while change and int(change) > maxChange:
 524                     changed = True
 525                     if self.verbose:
 526                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                     system("git update-ref %s \"%s^\"" % (ref, ref))
 528                     log = extractLogMessageFromGitCommit(ref)
 529                     settings =  extractSettingsGitLog(log)
 530
 531
 532                     depotPaths = settings['depot-paths']
 533                     change = settings['change']
 534
 535                 if changed:
 536                     print "%s rewound to %s" % (ref, change)
 537
 538         return True
 539
 540 class P4Submit(Command):
 541     def __init__(self):
 542         Command.__init__(self)
 543         self.options = [
 544                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                 optparse.make_option("--origin", dest="origin"),
 546                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 547         ]
 548         self.description = "Submit changes from git to the perforce depot."
 549         self.usage += " [name of git branch to submit into perforce depot]"
 550         self.interactive = True
 551         self.origin = ""
 552         self.detectRename = False
 553         self.verbose = False
 554         self.isWindows = (platform.system() == "Windows")
 555
 556     def check(self):
 557         if len(p4CmdList("opened ...")) > 0:
 558             die("You have files opened with perforce! Close them before starting the sync.")
 559
 560     # replaces everything between 'Description:' and the next P4 submit template field with the
 561     # commit message
 562     def prepareLogMessage(self, template, message):
 563         result = ""
 564
 565         inDescriptionSection = False
 566
 567         for line in template.split("\n"):
 568             if line.startswith("#"):
 569                 result += line + "\n"
 570                 continue
 571
 572             if inDescriptionSection:
 573                 if line.startswith("Files:"):
 574                     inDescriptionSection = False
 575                 else:
 576                     continue
 577             else:
 578                 if line.startswith("Description:"):
 579                     inDescriptionSection = True
 580                     line += "\n"
 581                     for messageLine in message.split("\n"):
 582                         line += "\t" + messageLine + "\n"
 583
 584             result += line + "\n"
 585
 586         return result
 587
 588     def prepareSubmitTemplate(self):
 589         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590         template = ""
 591         inFilesSection = False
 592         for line in p4_read_pipe_lines("change -o"):
 593             if line.endswith("\r\n"):
 594                 line = line[:-2] + "\n"
 595             if inFilesSection:
 596                 if line.startswith("\t"):
 597                     # path starts and ends with a tab
 598                     path = line[1:]
 599                     lastTab = path.rfind("\t")
 600                     if lastTab != -1:
 601                         path = path[:lastTab]
 602                         if not path.startswith(self.depotPath):
 603                             continue
 604                 else:
 605                     inFilesSection = False
 606             else:
 607                 if line.startswith("Files:"):
 608                     inFilesSection = True
 609
 610             template += line
 611
 612         return template
 613
 614     def applyCommit(self, id):
 615         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616         diffOpts = ("", "-M")[self.detectRename]
 617         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618         filesToAdd = set()
 619         filesToDelete = set()
 620         editedFiles = set()
 621         filesToChangeExecBit = {}
 622         for line in diff:
 623             diff = parseDiffTreeEntry(line)
 624             modifier = diff['status']
 625             path = diff['src']
 626             if modifier == "M":
 627                 p4_system("edit \"%s\"" % path)
 628                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                     filesToChangeExecBit[path] = diff['dst_mode']
 630                 editedFiles.add(path)
 631             elif modifier == "A":
 632                 filesToAdd.add(path)
 633                 filesToChangeExecBit[path] = diff['dst_mode']
 634                 if path in filesToDelete:
 635                     filesToDelete.remove(path)
 636             elif modifier == "D":
 637                 filesToDelete.add(path)
 638                 if path in filesToAdd:
 639                     filesToAdd.remove(path)
 640             elif modifier == "R":
 641                 src, dest = diff['src'], diff['dst']
 642                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                 p4_system("edit \"%s\"" % (dest))
 644                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                     filesToChangeExecBit[dest] = diff['dst_mode']
 646                 os.unlink(dest)
 647                 editedFiles.add(dest)
 648                 filesToDelete.add(src)
 649             else:
 650                 die("unknown modifier %s for %s" % (modifier, path))
 651
 652         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653         patchcmd = diffcmd + " | git apply "
 654         tryPatchCmd = patchcmd + "--check -"
 655         applyPatchCmd = patchcmd + "--check --apply -"
 656
 657         if os.system(tryPatchCmd) != 0:
 658             print "Unfortunately applying the change failed!"
 659             print "What do you want to do?"
 660             response = "x"
 661             while response != "s" and response != "a" and response != "w":
 662                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664             if response == "s":
 665                 print "Skipping! Good luck with the next patches..."
 666                 for f in editedFiles:
 667                     p4_system("revert \"%s\"" % f);
 668                 for f in filesToAdd:
 669                     system("rm %s" %f)
 670                 return
 671             elif response == "a":
 672                 os.system(applyPatchCmd)
 673                 if len(filesToAdd) > 0:
 674                     print "You may also want to call p4 add on the following files:"
 675                     print " ".join(filesToAdd)
 676                 if len(filesToDelete):
 677                     print "The following files should be scheduled for deletion with p4 delete:"
 678                     print " ".join(filesToDelete)
 679                 die("Please resolve and submit the conflict manually and "
 680                     + "continue afterwards with git-p4 submit --continue")
 681             elif response == "w":
 682                 system(diffcmd + " > patch.txt")
 683                 print "Patch saved to patch.txt in %s !" % self.clientPath
 684                 die("Please resolve and submit the conflict manually and "
 685                     "continue afterwards with git-p4 submit --continue")
 686
 687         system(applyPatchCmd)
 688
 689         for f in filesToAdd:
 690             p4_system("add \"%s\"" % f)
 691         for f in filesToDelete:
 692             p4_system("revert \"%s\"" % f)
 693             p4_system("delete \"%s\"" % f)
 694
 695         # Set/clear executable bits
 696         for f in filesToChangeExecBit.keys():
 697             mode = filesToChangeExecBit[f]
 698             setP4ExecBit(f, mode)
 699
 700         logMessage = extractLogMessageFromGitCommit(id)
 701         logMessage = logMessage.strip()
 702
 703         template = self.prepareSubmitTemplate()
 704
 705         if self.interactive:
 706             submitTemplate = self.prepareLogMessage(template, logMessage)
 707             if os.environ.has_key("P4DIFF"):
 708                 del(os.environ["P4DIFF"])
 709             diff = ""
 710             for editedFile in editedFiles:
 711                 diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713             newdiff = ""
 714             for newFile in filesToAdd:
 715                 newdiff += "==== new file ====\n"
 716                 newdiff += "--- /dev/null\n"
 717                 newdiff += "+++ %s\n" % newFile
 718                 f = open(newFile, "r")
 719                 for line in f.readlines():
 720                     newdiff += "+" + line
 721                 f.close()
 722
 723             separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725             [handle, fileName] = tempfile.mkstemp()
 726             tmpFile = os.fdopen(handle, "w+")
 727             if self.isWindows:
 728                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                 separatorLine = separatorLine.replace("\n", "\r\n")
 730                 newdiff = newdiff.replace("\n", "\r\n")
 731             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732             tmpFile.close()
 733             mtime = os.stat(fileName).st_mtime
 734             if os.environ.has_key("P4EDITOR"):
 735                 editor = os.environ.get("P4EDITOR")
 736             else:
 737                 editor = read_pipe("git var GIT_EDITOR").strip()
 738             system(editor + " " + fileName)
 739
 740             response = "y"
 741             if os.stat(fileName).st_mtime <= mtime:
 742                 response = "x"
 743                 while response != "y" and response != "n":
 744                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746             if response == "y":
 747                 tmpFile = open(fileName, "rb")
 748                 message = tmpFile.read()
 749                 tmpFile.close()
 750                 submitTemplate = message[:message.index(separatorLine)]
 751                 if self.isWindows:
 752                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                 p4_write_pipe("submit -i", submitTemplate)
 754             else:
 755                 for f in editedFiles:
 756                     p4_system("revert \"%s\"" % f);
 757                 for f in filesToAdd:
 758                     p4_system("revert \"%s\"" % f);
 759                     system("rm %s" %f)
 760
 761             os.remove(fileName)
 762         else:
 763             fileName = "submit.txt"
 764             file = open(fileName, "w+")
 765             file.write(self.prepareLogMessage(template, logMessage))
 766             file.close()
 767             print ("Perforce submit template written as %s. "
 768                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                    % (fileName, fileName))
 770
 771     def run(self, args):
 772         if len(args) == 0:
 773             self.master = currentGitBranch()
 774             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                 die("Detecting current git branch failed!")
 776         elif len(args) == 1:
 777             self.master = args[0]
 778         else:
 779             return False
 780
 781         allowSubmit = gitConfig("git-p4.allowSubmit")
 782         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783             die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785         [upstream, settings] = findUpstreamBranchPoint()
 786         self.depotPath = settings['depot-paths'][0]
 787         if len(self.origin) == 0:
 788             self.origin = upstream
 789
 790         if self.verbose:
 791             print "Origin branch is " + self.origin
 792
 793         if len(self.depotPath) == 0:
 794             print "Internal error: cannot locate perforce depot path from existing branches"
 795             sys.exit(128)
 796
 797         self.clientPath = p4Where(self.depotPath)
 798
 799         if len(self.clientPath) == 0:
 800             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801             sys.exit(128)
 802
 803         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804         self.oldWorkingDirectory = os.getcwd()
 805
 806         chdir(self.clientPath)
 807         print "Synchronizing p4 checkout..."
 808         p4_system("sync ...")
 809
 810         self.check()
 811
 812         commits = []
 813         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814             commits.append(line.strip())
 815         commits.reverse()
 816
 817         while len(commits) > 0:
 818             commit = commits[0]
 819             commits = commits[1:]
 820             self.applyCommit(commit)
 821             if not self.interactive:
 822                 break
 823
 824         if len(commits) == 0:
 825             print "All changes applied!"
 826             chdir(self.oldWorkingDirectory)
 827
 828             sync = P4Sync()
 829             sync.run([])
 830
 831             rebase = P4Rebase()
 832             rebase.rebase()
 833
 834         return True
 835
 836 class P4Sync(Command):
 837     delete_actions = ( "delete", "move/delete", "purge" )
 838
 839     def __init__(self):
 840         Command.__init__(self)
 841         self.options = [
 842                 optparse.make_option("--branch", dest="branch"),
 843                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 844                 optparse.make_option("--changesfile", dest="changesFile"),
 845                 optparse.make_option("--silent", dest="silent", action="store_true"),
 846                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 847                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 848                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 849                                      help="Import into refs/heads/ , not refs/remotes"),
 850                 optparse.make_option("--max-changes", dest="maxChanges"),
 851                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 852                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 853                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 854                                      help="Only sync files that are included in the Perforce Client Spec")
 855         ]
 856         self.description = """Imports from Perforce into a git repository.\n
 857     example:
 858     //depot/my/project/ -- to import the current head
 859     //depot/my/project/@all -- to import everything
 860     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 861
 862     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 863
 864         self.usage += " //depot/path[@revRange]"
 865         self.silent = False
 866         self.createdBranches = set()
 867         self.committedChanges = set()
 868         self.branch = ""
 869         self.detectBranches = False
 870         self.detectLabels = False
 871         self.changesFile = ""
 872         self.syncWithOrigin = True
 873         self.verbose = False
 874         self.importIntoRemotes = True
 875         self.maxChanges = ""
 876         self.isWindows = (platform.system() == "Windows")
 877         self.keepRepoPath = False
 878         self.depotPaths = None
 879         self.p4BranchesInGit = []
 880         self.cloneExclude = []
 881         self.useClientSpec = False
 882         self.clientSpecDirs = []
 883
 884         if gitConfig("git-p4.syncFromOrigin") == "false":
 885             self.syncWithOrigin = False
 886
 887     #
 888     # P4 wildcards are not allowed in filenames.  P4 complains
 889     # if you simply add them, but you can force it with "-f", in
 890     # which case it translates them into %xx encoding internally.
 891     # Search for and fix just these four characters.  Do % last so
 892     # that fixing it does not inadvertently create new %-escapes.
 893     #
 894     def wildcard_decode(self, path):
 895         # Cannot have * in a filename in windows; untested as to
 896         # what p4 would do in such a case.
 897         if not self.isWindows:
 898             path = path.replace("%2A", "*")
 899         path = path.replace("%23", "#") \
 900                    .replace("%40", "@") \
 901                    .replace("%25", "%")
 902         return path
 903
 904     def extractFilesFromCommit(self, commit):
 905         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 906                              for path in self.cloneExclude]
 907         files = []
 908         fnum = 0
 909         while commit.has_key("depotFile%s" % fnum):
 910             path =  commit["depotFile%s" % fnum]
 911
 912             if [p for p in self.cloneExclude
 913                 if path.startswith (p)]:
 914                 found = False
 915             else:
 916                 found = [p for p in self.depotPaths
 917                          if path.startswith (p)]
 918             if not found:
 919                 fnum = fnum + 1
 920                 continue
 921
 922             file = {}
 923             file["path"] = path
 924             file["rev"] = commit["rev%s" % fnum]
 925             file["action"] = commit["action%s" % fnum]
 926             file["type"] = commit["type%s" % fnum]
 927             files.append(file)
 928             fnum = fnum + 1
 929         return files
 930
 931     def stripRepoPath(self, path, prefixes):
 932         if self.useClientSpec:
 933
 934             # if using the client spec, we use the output directory
 935             # specified in the client.  For example, a view
 936             #   //depot/foo/branch/... //client/branch/foo/...
 937             # will end up putting all foo/branch files into
 938             #  branch/foo/
 939             for val in self.clientSpecDirs:
 940                 if path.startswith(val[0]):
 941                     # replace the depot path with the client path
 942                     path = path.replace(val[0], val[1][1])
 943                     # now strip out the client (//client/...)
 944                     path = re.sub("^(//[^/]+/)", '', path)
 945                     # the rest is all path
 946                     return path
 947
 948         if self.keepRepoPath:
 949             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 950
 951         for p in prefixes:
 952             if path.startswith(p):
 953                 path = path[len(p):]
 954
 955         return path
 956
 957     def splitFilesIntoBranches(self, commit):
 958         branches = {}
 959         fnum = 0
 960         while commit.has_key("depotFile%s" % fnum):
 961             path =  commit["depotFile%s" % fnum]
 962             found = [p for p in self.depotPaths
 963                      if path.startswith (p)]
 964             if not found:
 965                 fnum = fnum + 1
 966                 continue
 967
 968             file = {}
 969             file["path"] = path
 970             file["rev"] = commit["rev%s" % fnum]
 971             file["action"] = commit["action%s" % fnum]
 972             file["type"] = commit["type%s" % fnum]
 973             fnum = fnum + 1
 974
 975             relPath = self.stripRepoPath(path, self.depotPaths)
 976
 977             for branch in self.knownBranches.keys():
 978
 979                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 980                 if relPath.startswith(branch + "/"):
 981                     if branch not in branches:
 982                         branches[branch] = []
 983                     branches[branch].append(file)
 984                     break
 985
 986         return branches
 987
 988     # output one file from the P4 stream
 989     # - helper for streamP4Files
 990
 991     def streamOneP4File(self, file, contents):
 992         if file["type"] == "apple":
 993             print "\nfile %s is a strange apple file that forks. Ignoring" % \
 994                 file['depotFile']
 995             return
 996
 997         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 998         relPath = self.wildcard_decode(relPath)
 999         if verbose:
1000             sys.stderr.write("%s\n" % relPath)
1001
1002         mode = "644"
1003         if isP4Exec(file["type"]):
1004             mode = "755"
1005         elif file["type"] == "symlink":
1006             mode = "120000"
1007             # p4 print on a symlink contains "target\n", so strip it off
1008             data = ''.join(contents)
1009             contents = [data[:-1]]
1010
1011         if self.isWindows and file["type"].endswith("text"):
1012             mangled = []
1013             for data in contents:
1014                 data = data.replace("\r\n", "\n")
1015                 mangled.append(data)
1016             contents = mangled
1017
1018         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1019             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1020         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1021             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1022
1023         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1024
1025         # total length...
1026         length = 0
1027         for d in contents:
1028             length = length + len(d)
1029
1030         self.gitStream.write("data %d\n" % length)
1031         for d in contents:
1032             self.gitStream.write(d)
1033         self.gitStream.write("\n")
1034
1035     def streamOneP4Deletion(self, file):
1036         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1037         if verbose:
1038             sys.stderr.write("delete %s\n" % relPath)
1039         self.gitStream.write("D %s\n" % relPath)
1040
1041     # handle another chunk of streaming data
1042     def streamP4FilesCb(self, marshalled):
1043
1044         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1045             # start of a new file - output the old one first
1046             self.streamOneP4File(self.stream_file, self.stream_contents)
1047             self.stream_file = {}
1048             self.stream_contents = []
1049             self.stream_have_file_info = False
1050
1051         # pick up the new file information... for the
1052         # 'data' field we need to append to our array
1053         for k in marshalled.keys():
1054             if k == 'data':
1055                 self.stream_contents.append(marshalled['data'])
1056             else:
1057                 self.stream_file[k] = marshalled[k]
1058
1059         self.stream_have_file_info = True
1060
1061     # Stream directly from "p4 files" into "git fast-import"
1062     def streamP4Files(self, files):
1063         filesForCommit = []
1064         filesToRead = []
1065         filesToDelete = []
1066
1067         for f in files:
1068             includeFile = True
1069             for val in self.clientSpecDirs:
1070                 if f['path'].startswith(val[0]):
1071                     if val[1][0] <= 0:
1072                         includeFile = False
1073                     break
1074
1075             if includeFile:
1076                 filesForCommit.append(f)
1077                 if f['action'] in self.delete_actions:
1078                     filesToDelete.append(f)
1079                 else:
1080                     filesToRead.append(f)
1081
1082         # deleted files...
1083         for f in filesToDelete:
1084             self.streamOneP4Deletion(f)
1085
1086         if len(filesToRead) > 0:
1087             self.stream_file = {}
1088             self.stream_contents = []
1089             self.stream_have_file_info = False
1090
1091             # curry self argument
1092             def streamP4FilesCbSelf(entry):
1093                 self.streamP4FilesCb(entry)
1094
1095             p4CmdList("-x - print",
1096                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1097                                                   for f in filesToRead]),
1098                 cb=streamP4FilesCbSelf)
1099
1100             # do the last chunk
1101             if self.stream_file.has_key('depotFile'):
1102                 self.streamOneP4File(self.stream_file, self.stream_contents)
1103
1104     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1105         epoch = details["time"]
1106         author = details["user"]
1107         self.branchPrefixes = branchPrefixes
1108
1109         if self.verbose:
1110             print "commit into %s" % branch
1111
1112         # start with reading files; if that fails, we should not
1113         # create a commit.
1114         new_files = []
1115         for f in files:
1116             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1117                 new_files.append (f)
1118             else:
1119                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1120
1121         self.gitStream.write("commit %s\n" % branch)
1122 #        gitStream.write("mark :%s\n" % details["change"])
1123         self.committedChanges.add(int(details["change"]))
1124         committer = ""
1125         if author not in self.users:
1126             self.getUserMapFromPerforceServer()
1127         if author in self.users:
1128             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1129         else:
1130             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1131
1132         self.gitStream.write("committer %s\n" % committer)
1133
1134         self.gitStream.write("data <<EOT\n")
1135         self.gitStream.write(details["desc"])
1136         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1137                              % (','.join (branchPrefixes), details["change"]))
1138         if len(details['options']) > 0:
1139             self.gitStream.write(": options = %s" % details['options'])
1140         self.gitStream.write("]\nEOT\n\n")
1141
1142         if len(parent) > 0:
1143             if self.verbose:
1144                 print "parent %s" % parent
1145             self.gitStream.write("from %s\n" % parent)
1146
1147         self.streamP4Files(new_files)
1148         self.gitStream.write("\n")
1149
1150         change = int(details["change"])
1151
1152         if self.labels.has_key(change):
1153             label = self.labels[change]
1154             labelDetails = label[0]
1155             labelRevisions = label[1]
1156             if self.verbose:
1157                 print "Change %s is labelled %s" % (change, labelDetails)
1158
1159             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1160                                                     for p in branchPrefixes]))
1161
1162             if len(files) == len(labelRevisions):
1163
1164                 cleanedFiles = {}
1165                 for info in files:
1166                     if info["action"] in self.delete_actions:
1167                         continue
1168                     cleanedFiles[info["depotFile"]] = info["rev"]
1169
1170                 if cleanedFiles == labelRevisions:
1171                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1172                     self.gitStream.write("from %s\n" % branch)
1173
1174                     owner = labelDetails["Owner"]
1175                     tagger = ""
1176                     if author in self.users:
1177                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1178                     else:
1179                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1180                     self.gitStream.write("tagger %s\n" % tagger)
1181                     self.gitStream.write("data <<EOT\n")
1182                     self.gitStream.write(labelDetails["Description"])
1183                     self.gitStream.write("EOT\n\n")
1184
1185                 else:
1186                     if not self.silent:
1187                         print ("Tag %s does not match with change %s: files do not match."
1188                                % (labelDetails["label"], change))
1189
1190             else:
1191                 if not self.silent:
1192                     print ("Tag %s does not match with change %s: file count is different."
1193                            % (labelDetails["label"], change))
1194
1195     def getUserCacheFilename(self):
1196         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1197         return home + "/.gitp4-usercache.txt"
1198
1199     def getUserMapFromPerforceServer(self):
1200         if self.userMapFromPerforceServer:
1201             return
1202         self.users = {}
1203
1204         for output in p4CmdList("users"):
1205             if not output.has_key("User"):
1206                 continue
1207             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1208
1209
1210         s = ''
1211         for (key, val) in self.users.items():
1212             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1213
1214         open(self.getUserCacheFilename(), "wb").write(s)
1215         self.userMapFromPerforceServer = True
1216
1217     def loadUserMapFromCache(self):
1218         self.users = {}
1219         self.userMapFromPerforceServer = False
1220         try:
1221             cache = open(self.getUserCacheFilename(), "rb")
1222             lines = cache.readlines()
1223             cache.close()
1224             for line in lines:
1225                 entry = line.strip().split("\t")
1226                 self.users[entry[0]] = entry[1]
1227         except IOError:
1228             self.getUserMapFromPerforceServer()
1229
1230     def getLabels(self):
1231         self.labels = {}
1232
1233         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1234         if len(l) > 0 and not self.silent:
1235             print "Finding files belonging to labels in %s" % `self.depotPaths`
1236
1237         for output in l:
1238             label = output["label"]
1239             revisions = {}
1240             newestChange = 0
1241             if self.verbose:
1242                 print "Querying files for label %s" % label
1243             for file in p4CmdList("files "
1244                                   +  ' '.join (["%s...@%s" % (p, label)
1245                                                 for p in self.depotPaths])):
1246                 revisions[file["depotFile"]] = file["rev"]
1247                 change = int(file["change"])
1248                 if change > newestChange:
1249                     newestChange = change
1250
1251             self.labels[newestChange] = [output, revisions]
1252
1253         if self.verbose:
1254             print "Label changes: %s" % self.labels.keys()
1255
1256     def guessProjectName(self):
1257         for p in self.depotPaths:
1258             if p.endswith("/"):
1259                 p = p[:-1]
1260             p = p[p.strip().rfind("/") + 1:]
1261             if not p.endswith("/"):
1262                p += "/"
1263             return p
1264
1265     def getBranchMapping(self):
1266         lostAndFoundBranches = set()
1267
1268         for info in p4CmdList("branches"):
1269             details = p4Cmd("branch -o %s" % info["branch"])
1270             viewIdx = 0
1271             while details.has_key("View%s" % viewIdx):
1272                 paths = details["View%s" % viewIdx].split(" ")
1273                 viewIdx = viewIdx + 1
1274                 # require standard //depot/foo/... //depot/bar/... mapping
1275                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1276                     continue
1277                 source = paths[0]
1278                 destination = paths[1]
1279                 ## HACK
1280                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1281                     source = source[len(self.depotPaths[0]):-4]
1282                     destination = destination[len(self.depotPaths[0]):-4]
1283
1284                     if destination in self.knownBranches:
1285                         if not self.silent:
1286                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1287                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1288                         continue
1289
1290                     self.knownBranches[destination] = source
1291
1292                     lostAndFoundBranches.discard(destination)
1293
1294                     if source not in self.knownBranches:
1295                         lostAndFoundBranches.add(source)
1296
1297
1298         for branch in lostAndFoundBranches:
1299             self.knownBranches[branch] = branch
1300
1301     def getBranchMappingFromGitBranches(self):
1302         branches = p4BranchesInGit(self.importIntoRemotes)
1303         for branch in branches.keys():
1304             if branch == "master":
1305                 branch = "main"
1306             else:
1307                 branch = branch[len(self.projectName):]
1308             self.knownBranches[branch] = branch
1309
1310     def listExistingP4GitBranches(self):
1311         # branches holds mapping from name to commit
1312         branches = p4BranchesInGit(self.importIntoRemotes)
1313         self.p4BranchesInGit = branches.keys()
1314         for branch in branches.keys():
1315             self.initialParents[self.refPrefix + branch] = branches[branch]
1316
1317     def updateOptionDict(self, d):
1318         option_keys = {}
1319         if self.keepRepoPath:
1320             option_keys['keepRepoPath'] = 1
1321
1322         d["options"] = ' '.join(sorted(option_keys.keys()))
1323
1324     def readOptions(self, d):
1325         self.keepRepoPath = (d.has_key('options')
1326                              and ('keepRepoPath' in d['options']))
1327
1328     def gitRefForBranch(self, branch):
1329         if branch == "main":
1330             return self.refPrefix + "master"
1331
1332         if len(branch) <= 0:
1333             return branch
1334
1335         return self.refPrefix + self.projectName + branch
1336
1337     def gitCommitByP4Change(self, ref, change):
1338         if self.verbose:
1339             print "looking in ref " + ref + " for change %s using bisect..." % change
1340
1341         earliestCommit = ""
1342         latestCommit = parseRevision(ref)
1343
1344         while True:
1345             if self.verbose:
1346                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1347             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1348             if len(next) == 0:
1349                 if self.verbose:
1350                     print "argh"
1351                 return ""
1352             log = extractLogMessageFromGitCommit(next)
1353             settings = extractSettingsGitLog(log)
1354             currentChange = int(settings['change'])
1355             if self.verbose:
1356                 print "current change %s" % currentChange
1357
1358             if currentChange == change:
1359                 if self.verbose:
1360                     print "found %s" % next
1361                 return next
1362
1363             if currentChange < change:
1364                 earliestCommit = "^%s" % next
1365             else:
1366                 latestCommit = "%s" % next
1367
1368         return ""
1369
1370     def importNewBranch(self, branch, maxChange):
1371         # make fast-import flush all changes to disk and update the refs using the checkpoint
1372         # command so that we can try to find the branch parent in the git history
1373         self.gitStream.write("checkpoint\n\n");
1374         self.gitStream.flush();
1375         branchPrefix = self.depotPaths[0] + branch + "/"
1376         range = "@1,%s" % maxChange
1377         #print "prefix" + branchPrefix
1378         changes = p4ChangesForPaths([branchPrefix], range)
1379         if len(changes) <= 0:
1380             return False
1381         firstChange = changes[0]
1382         #print "first change in branch: %s" % firstChange
1383         sourceBranch = self.knownBranches[branch]
1384         sourceDepotPath = self.depotPaths[0] + sourceBranch
1385         sourceRef = self.gitRefForBranch(sourceBranch)
1386         #print "source " + sourceBranch
1387
1388         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1389         #print "branch parent: %s" % branchParentChange
1390         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1391         if len(gitParent) > 0:
1392             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1393             #print "parent git commit: %s" % gitParent
1394
1395         self.importChanges(changes)
1396         return True
1397
1398     def importChanges(self, changes):
1399         cnt = 1
1400         for change in changes:
1401             description = p4Cmd("describe %s" % change)
1402             self.updateOptionDict(description)
1403
1404             if not self.silent:
1405                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1406                 sys.stdout.flush()
1407             cnt = cnt + 1
1408
1409             try:
1410                 if self.detectBranches:
1411                     branches = self.splitFilesIntoBranches(description)
1412                     for branch in branches.keys():
1413                         ## HACK  --hwn
1414                         branchPrefix = self.depotPaths[0] + branch + "/"
1415
1416                         parent = ""
1417
1418                         filesForCommit = branches[branch]
1419
1420                         if self.verbose:
1421                             print "branch is %s" % branch
1422
1423                         self.updatedBranches.add(branch)
1424
1425                         if branch not in self.createdBranches:
1426                             self.createdBranches.add(branch)
1427                             parent = self.knownBranches[branch]
1428                             if parent == branch:
1429                                 parent = ""
1430                             else:
1431                                 fullBranch = self.projectName + branch
1432                                 if fullBranch not in self.p4BranchesInGit:
1433                                     if not self.silent:
1434                                         print("\n    Importing new branch %s" % fullBranch);
1435                                     if self.importNewBranch(branch, change - 1):
1436                                         parent = ""
1437                                         self.p4BranchesInGit.append(fullBranch)
1438                                     if not self.silent:
1439                                         print("\n    Resuming with change %s" % change);
1440
1441                                 if self.verbose:
1442                                     print "parent determined through known branches: %s" % parent
1443
1444                         branch = self.gitRefForBranch(branch)
1445                         parent = self.gitRefForBranch(parent)
1446
1447                         if self.verbose:
1448                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1449
1450                         if len(parent) == 0 and branch in self.initialParents:
1451                             parent = self.initialParents[branch]
1452                             del self.initialParents[branch]
1453
1454                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1455                 else:
1456                     files = self.extractFilesFromCommit(description)
1457                     self.commit(description, files, self.branch, self.depotPaths,
1458                                 self.initialParent)
1459                     self.initialParent = ""
1460             except IOError:
1461                 print self.gitError.read()
1462                 sys.exit(1)
1463
1464     def importHeadRevision(self, revision):
1465         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1466
1467         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1468         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1469                            % (' '.join(self.depotPaths), revision))
1470         details["change"] = revision
1471         newestRevision = 0
1472
1473         fileCnt = 0
1474         for info in p4CmdList("files "
1475                               +  ' '.join(["%s...%s"
1476                                            % (p, revision)
1477                                            for p in self.depotPaths])):
1478
1479             if 'code' in info and info['code'] == 'error':
1480                 sys.stderr.write("p4 returned an error: %s\n"
1481                                  % info['data'])
1482                 if info['data'].find("must refer to client") >= 0:
1483                     sys.stderr.write("This particular p4 error is misleading.\n")
1484                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1485                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1486                 sys.exit(1)
1487             if 'p4ExitCode' in info:
1488                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1489                 sys.exit(1)
1490
1491
1492             change = int(info["change"])
1493             if change > newestRevision:
1494                 newestRevision = change
1495
1496             if info["action"] in self.delete_actions:
1497                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1498                 #fileCnt = fileCnt + 1
1499                 continue
1500
1501             for prop in ["depotFile", "rev", "action", "type" ]:
1502                 details["%s%s" % (prop, fileCnt)] = info[prop]
1503
1504             fileCnt = fileCnt + 1
1505
1506         details["change"] = newestRevision
1507         self.updateOptionDict(details)
1508         try:
1509             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1510         except IOError:
1511             print "IO error with git fast-import. Is your git version recent enough?"
1512             print self.gitError.read()
1513
1514
1515     def getClientSpec(self):
1516         specList = p4CmdList( "client -o" )
1517         temp = {}
1518         for entry in specList:
1519             for k,v in entry.iteritems():
1520                 if k.startswith("View"):
1521
1522                     # p4 has these %%1 to %%9 arguments in specs to
1523                     # reorder paths; which we can't handle (yet :)
1524                     if re.match('%%\d', v) != None:
1525                         print "Sorry, can't handle %%n arguments in client specs"
1526                         sys.exit(1)
1527
1528                     if v.startswith('"'):
1529                         start = 1
1530                     else:
1531                         start = 0
1532                     index = v.find("...")
1533
1534                     # save the "client view"; i.e the RHS of the view
1535                     # line that tells the client where to put the
1536                     # files for this view.
1537                     cv = v[index+3:].strip() # +3 to remove previous '...'
1538
1539                     # if the client view doesn't end with a
1540                     # ... wildcard, then we're going to mess up the
1541                     # output directory, so fail gracefully.
1542                     if not cv.endswith('...'):
1543                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1544                         sys.exit(1)
1545                     cv=cv[:-3]
1546
1547                     # now save the view; +index means included, -index
1548                     # means it should be filtered out.
1549                     v = v[start:index]
1550                     if v.startswith("-"):
1551                         v = v[1:]
1552                         include = -len(v)
1553                     else:
1554                         include = len(v)
1555
1556                     temp[v] = (include, cv)
1557
1558         self.clientSpecDirs = temp.items()
1559         self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1560
1561     def run(self, args):
1562         self.depotPaths = []
1563         self.changeRange = ""
1564         self.initialParent = ""
1565         self.previousDepotPaths = []
1566
1567         # map from branch depot path to parent branch
1568         self.knownBranches = {}
1569         self.initialParents = {}
1570         self.hasOrigin = originP4BranchesExist()
1571         if not self.syncWithOrigin:
1572             self.hasOrigin = False
1573
1574         if self.importIntoRemotes:
1575             self.refPrefix = "refs/remotes/p4/"
1576         else:
1577             self.refPrefix = "refs/heads/p4/"
1578
1579         if self.syncWithOrigin and self.hasOrigin:
1580             if not self.silent:
1581                 print "Syncing with origin first by calling git fetch origin"
1582             system("git fetch origin")
1583
1584         if len(self.branch) == 0:
1585             self.branch = self.refPrefix + "master"
1586             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1587                 system("git update-ref %s refs/heads/p4" % self.branch)
1588                 system("git branch -D p4");
1589             # create it /after/ importing, when master exists
1590             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1591                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1592
1593         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1594             self.getClientSpec()
1595
1596         # TODO: should always look at previous commits,
1597         # merge with previous imports, if possible.
1598         if args == []:
1599             if self.hasOrigin:
1600                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1601             self.listExistingP4GitBranches()
1602
1603             if len(self.p4BranchesInGit) > 1:
1604                 if not self.silent:
1605                     print "Importing from/into multiple branches"
1606                 self.detectBranches = True
1607
1608             if self.verbose:
1609                 print "branches: %s" % self.p4BranchesInGit
1610
1611             p4Change = 0
1612             for branch in self.p4BranchesInGit:
1613                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1614
1615                 settings = extractSettingsGitLog(logMsg)
1616
1617                 self.readOptions(settings)
1618                 if (settings.has_key('depot-paths')
1619                     and settings.has_key ('change')):
1620                     change = int(settings['change']) + 1
1621                     p4Change = max(p4Change, change)
1622
1623                     depotPaths = sorted(settings['depot-paths'])
1624                     if self.previousDepotPaths == []:
1625                         self.previousDepotPaths = depotPaths
1626                     else:
1627                         paths = []
1628                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1629                             for i in range(0, min(len(cur), len(prev))):
1630                                 if cur[i] <> prev[i]:
1631                                     i = i - 1
1632                                     break
1633
1634                             paths.append (cur[:i + 1])
1635
1636                         self.previousDepotPaths = paths
1637
1638             if p4Change > 0:
1639                 self.depotPaths = sorted(self.previousDepotPaths)
1640                 self.changeRange = "@%s,#head" % p4Change
1641                 if not self.detectBranches:
1642                     self.initialParent = parseRevision(self.branch)
1643                 if not self.silent and not self.detectBranches:
1644                     print "Performing incremental import into %s git branch" % self.branch
1645
1646         if not self.branch.startswith("refs/"):
1647             self.branch = "refs/heads/" + self.branch
1648
1649         if len(args) == 0 and self.depotPaths:
1650             if not self.silent:
1651                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1652         else:
1653             if self.depotPaths and self.depotPaths != args:
1654                 print ("previous import used depot path %s and now %s was specified. "
1655                        "This doesn't work!" % (' '.join (self.depotPaths),
1656                                                ' '.join (args)))
1657                 sys.exit(1)
1658
1659             self.depotPaths = sorted(args)
1660
1661         revision = ""
1662         self.users = {}
1663
1664         newPaths = []
1665         for p in self.depotPaths:
1666             if p.find("@") != -1:
1667                 atIdx = p.index("@")
1668                 self.changeRange = p[atIdx:]
1669                 if self.changeRange == "@all":
1670                     self.changeRange = ""
1671                 elif ',' not in self.changeRange:
1672                     revision = self.changeRange
1673                     self.changeRange = ""
1674                 p = p[:atIdx]
1675             elif p.find("#") != -1:
1676                 hashIdx = p.index("#")
1677                 revision = p[hashIdx:]
1678                 p = p[:hashIdx]
1679             elif self.previousDepotPaths == []:
1680                 revision = "#head"
1681
1682             p = re.sub ("\.\.\.$", "", p)
1683             if not p.endswith("/"):
1684                 p += "/"
1685
1686             newPaths.append(p)
1687
1688         self.depotPaths = newPaths
1689
1690
1691         self.loadUserMapFromCache()
1692         self.labels = {}
1693         if self.detectLabels:
1694             self.getLabels();
1695
1696         if self.detectBranches:
1697             ## FIXME - what's a P4 projectName ?
1698             self.projectName = self.guessProjectName()
1699
1700             if self.hasOrigin:
1701                 self.getBranchMappingFromGitBranches()
1702             else:
1703                 self.getBranchMapping()
1704             if self.verbose:
1705                 print "p4-git branches: %s" % self.p4BranchesInGit
1706                 print "initial parents: %s" % self.initialParents
1707             for b in self.p4BranchesInGit:
1708                 if b != "master":
1709
1710                     ## FIXME
1711                     b = b[len(self.projectName):]
1712                 self.createdBranches.add(b)
1713
1714         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1715
1716         importProcess = subprocess.Popen(["git", "fast-import"],
1717                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1718                                          stderr=subprocess.PIPE);
1719         self.gitOutput = importProcess.stdout
1720         self.gitStream = importProcess.stdin
1721         self.gitError = importProcess.stderr
1722
1723         if revision:
1724             self.importHeadRevision(revision)
1725         else:
1726             changes = []
1727
1728             if len(self.changesFile) > 0:
1729                 output = open(self.changesFile).readlines()
1730                 changeSet = set()
1731                 for line in output:
1732                     changeSet.add(int(line))
1733
1734                 for change in changeSet:
1735                     changes.append(change)
1736
1737                 changes.sort()
1738             else:
1739                 if not self.p4BranchesInGit:
1740                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
1741                 if self.verbose:
1742                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1743                                                               self.changeRange)
1744                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1745
1746                 if len(self.maxChanges) > 0:
1747                     changes = changes[:min(int(self.maxChanges), len(changes))]
1748
1749             if len(changes) == 0:
1750                 if not self.silent:
1751                     print "No changes to import!"
1752                 return True
1753
1754             if not self.silent and not self.detectBranches:
1755                 print "Import destination: %s" % self.branch
1756
1757             self.updatedBranches = set()
1758
1759             self.importChanges(changes)
1760
1761             if not self.silent:
1762                 print ""
1763                 if len(self.updatedBranches) > 0:
1764                     sys.stdout.write("Updated branches: ")
1765                     for b in self.updatedBranches:
1766                         sys.stdout.write("%s " % b)
1767                     sys.stdout.write("\n")
1768
1769         self.gitStream.close()
1770         if importProcess.wait() != 0:
1771             die("fast-import failed: %s" % self.gitError.read())
1772         self.gitOutput.close()
1773         self.gitError.close()
1774
1775         return True
1776
1777 class P4Rebase(Command):
1778     def __init__(self):
1779         Command.__init__(self)
1780         self.options = [ ]
1781         self.description = ("Fetches the latest revision from perforce and "
1782                             + "rebases the current work (branch) against it")
1783         self.verbose = False
1784
1785     def run(self, args):
1786         sync = P4Sync()
1787         sync.run([])
1788
1789         return self.rebase()
1790
1791     def rebase(self):
1792         if os.system("git update-index --refresh") != 0:
1793             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1794         if len(read_pipe("git diff-index HEAD --")) > 0:
1795             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1796
1797         [upstream, settings] = findUpstreamBranchPoint()
1798         if len(upstream) == 0:
1799             die("Cannot find upstream branchpoint for rebase")
1800
1801         # the branchpoint may be p4/foo~3, so strip off the parent
1802         upstream = re.sub("~[0-9]+$", "", upstream)
1803
1804         print "Rebasing the current branch onto %s" % upstream
1805         oldHead = read_pipe("git rev-parse HEAD").strip()
1806         system("git rebase %s" % upstream)
1807         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1808         return True
1809
1810 class P4Clone(P4Sync):
1811     def __init__(self):
1812         P4Sync.__init__(self)
1813         self.description = "Creates a new git repository and imports from Perforce into it"
1814         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1815         self.options += [
1816             optparse.make_option("--destination", dest="cloneDestination",
1817                                  action='store', default=None,
1818                                  help="where to leave result of the clone"),
1819             optparse.make_option("-/", dest="cloneExclude",
1820                                  action="append", type="string",
1821                                  help="exclude depot path"),
1822             optparse.make_option("--bare", dest="cloneBare",
1823                                  action="store_true", default=False),
1824         ]
1825         self.cloneDestination = None
1826         self.needsGit = False
1827         self.cloneBare = False
1828
1829     # This is required for the "append" cloneExclude action
1830     def ensure_value(self, attr, value):
1831         if not hasattr(self, attr) or getattr(self, attr) is None:
1832             setattr(self, attr, value)
1833         return getattr(self, attr)
1834
1835     def defaultDestination(self, args):
1836         ## TODO: use common prefix of args?
1837         depotPath = args[0]
1838         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1839         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1840         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1841         depotDir = re.sub(r"/$", "", depotDir)
1842         return os.path.split(depotDir)[1]
1843
1844     def run(self, args):
1845         if len(args) < 1:
1846             return False
1847
1848         if self.keepRepoPath and not self.cloneDestination:
1849             sys.stderr.write("Must specify destination for --keep-path\n")
1850             sys.exit(1)
1851
1852         depotPaths = args
1853
1854         if not self.cloneDestination and len(depotPaths) > 1:
1855             self.cloneDestination = depotPaths[-1]
1856             depotPaths = depotPaths[:-1]
1857
1858         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1859         for p in depotPaths:
1860             if not p.startswith("//"):
1861                 return False
1862
1863         if not self.cloneDestination:
1864             self.cloneDestination = self.defaultDestination(args)
1865
1866         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1867
1868         if not os.path.exists(self.cloneDestination):
1869             os.makedirs(self.cloneDestination)
1870         chdir(self.cloneDestination)
1871
1872         init_cmd = [ "git", "init" ]
1873         if self.cloneBare:
1874             init_cmd.append("--bare")
1875         subprocess.check_call(init_cmd)
1876
1877         if not P4Sync.run(self, depotPaths):
1878             return False
1879         if self.branch != "master":
1880             if self.importIntoRemotes:
1881                 masterbranch = "refs/remotes/p4/master"
1882             else:
1883                 masterbranch = "refs/heads/p4/master"
1884             if gitBranchExists(masterbranch):
1885                 system("git branch master %s" % masterbranch)
1886                 if not self.cloneBare:
1887                     system("git checkout -f")
1888             else:
1889                 print "Could not detect main branch. No checkout/master branch created."
1890
1891         return True
1892
1893 class P4Branches(Command):
1894     def __init__(self):
1895         Command.__init__(self)
1896         self.options = [ ]
1897         self.description = ("Shows the git branches that hold imports and their "
1898                             + "corresponding perforce depot paths")
1899         self.verbose = False
1900
1901     def run(self, args):
1902         if originP4BranchesExist():
1903             createOrUpdateBranchesFromOrigin()
1904
1905         cmdline = "git rev-parse --symbolic "
1906         cmdline += " --remotes"
1907
1908         for line in read_pipe_lines(cmdline):
1909             line = line.strip()
1910
1911             if not line.startswith('p4/') or line == "p4/HEAD":
1912                 continue
1913             branch = line
1914
1915             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1916             settings = extractSettingsGitLog(log)
1917
1918             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1919         return True
1920
1921 class HelpFormatter(optparse.IndentedHelpFormatter):
1922     def __init__(self):
1923         optparse.IndentedHelpFormatter.__init__(self)
1924
1925     def format_description(self, description):
1926         if description:
1927             return description + "\n"
1928         else:
1929             return ""
1930
1931 def printUsage(commands):
1932     print "usage: %s <command> [options]" % sys.argv[0]
1933     print ""
1934     print "valid commands: %s" % ", ".join(commands)
1935     print ""
1936     print "Try %s <command> --help for command specific help." % sys.argv[0]
1937     print ""
1938
1939 commands = {
1940     "debug" : P4Debug,
1941     "submit" : P4Submit,
1942     "commit" : P4Submit,
1943     "sync" : P4Sync,
1944     "rebase" : P4Rebase,
1945     "clone" : P4Clone,
1946     "rollback" : P4RollBack,
1947     "branches" : P4Branches
1948 }
1949
1950
1951 def main():
1952     if len(sys.argv[1:]) == 0:
1953         printUsage(commands.keys())
1954         sys.exit(2)
1955
1956     cmd = ""
1957     cmdName = sys.argv[1]
1958     try:
1959         klass = commands[cmdName]
1960         cmd = klass()
1961     except KeyError:
1962         print "unknown command %s" % cmdName
1963         print ""
1964         printUsage(commands.keys())
1965         sys.exit(2)
1966
1967     options = cmd.options
1968     cmd.gitdir = os.environ.get("GIT_DIR", None)
1969
1970     args = sys.argv[2:]
1971
1972     if len(options) > 0:
1973         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1974
1975         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1976                                        options,
1977                                        description = cmd.description,
1978                                        formatter = HelpFormatter())
1979
1980         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1981     global verbose
1982     verbose = cmd.verbose
1983     if cmd.needsGit:
1984         if cmd.gitdir == None:
1985             cmd.gitdir = os.path.abspath(".git")
1986             if not isValidGitDir(cmd.gitdir):
1987                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1988                 if os.path.exists(cmd.gitdir):
1989                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1990                     if len(cdup) > 0:
1991                         chdir(cdup);
1992
1993         if not isValidGitDir(cmd.gitdir):
1994             if isValidGitDir(cmd.gitdir + "/.git"):
1995                 cmd.gitdir += "/.git"
1996             else:
1997                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1998
1999         os.environ["GIT_DIR"] = cmd.gitdir
2000
2001     if not cmd.run(args):
2002         parser.print_help()
2003
2004
2005 if __name__ == '__main__':
2006     main()