contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     if os.name == 'nt':
  56         os.environ['PWD']=dir
  57     os.chdir(dir)
  58
  59 def die(msg):
  60     if verbose:
  61         raise Exception(msg)
  62     else:
  63         sys.stderr.write(msg + "\n")
  64         sys.exit(1)
  65
  66 def write_pipe(c, str):
  67     if verbose:
  68         sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70     pipe = os.popen(c, 'w')
  71     val = pipe.write(str)
  72     if pipe.close():
  73         die('Command failed: %s' % c)
  74
  75     return val
  76
  77 def p4_write_pipe(c, str):
  78     real_cmd = p4_build_cmd(c)
  79     return write_pipe(real_cmd, str)
  80
  81 def read_pipe(c, ignore_error=False):
  82     if verbose:
  83         sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85     pipe = os.popen(c, 'rb')
  86     val = pipe.read()
  87     if pipe.close() and not ignore_error:
  88         die('Command failed: %s' % c)
  89
  90     return val
  91
  92 def p4_read_pipe(c, ignore_error=False):
  93     real_cmd = p4_build_cmd(c)
  94     return read_pipe(real_cmd, ignore_error)
  95
  96 def read_pipe_lines(c):
  97     if verbose:
  98         sys.stderr.write('Reading pipe: %s\n' % c)
  99     ## todo: check return status
 100     pipe = os.popen(c, 'rb')
 101     val = pipe.readlines()
 102     if pipe.close():
 103         die('Command failed: %s' % c)
 104
 105     return val
 106
 107 def p4_read_pipe_lines(c):
 108     """Specifically invoke p4 on the command supplied. """
 109     real_cmd = p4_build_cmd(c)
 110     return read_pipe_lines(real_cmd)
 111
 112 def system(cmd):
 113     if verbose:
 114         sys.stderr.write("executing %s\n" % cmd)
 115     if os.system(cmd) != 0:
 116         die("command failed: %s" % cmd)
 117
 118 def p4_system(cmd):
 119     """Specifically invoke p4 as the system command. """
 120     real_cmd = p4_build_cmd(cmd)
 121     return system(real_cmd)
 122
 123 def isP4Exec(kind):
 124     """Determine if a Perforce 'kind' should have execute permission
 125
 126     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128     a plus sign, it is also executable"""
 129     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131 def setP4ExecBit(file, mode):
 132     # Reopens an already open file and changes the execute bit to match
 133     # the execute bit setting in the passed in mode.
 134
 135     p4Type = "+x"
 136
 137     if not isModeExec(mode):
 138         p4Type = getP4OpenedType(file)
 139         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141         if p4Type[-1] == "+":
 142             p4Type = p4Type[0:-1]
 143
 144     p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146 def getP4OpenedType(file):
 147     # Returns the perforce file type for the given file.
 148
 149     result = p4_read_pipe("opened %s" % file)
 150     match = re.match(".*\((.+)\)\r?$", result)
 151     if match:
 152         return match.group(1)
 153     else:
 154         die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156 def diffTreePattern():
 157     # This is a simple generator for the diff tree regex pattern. This could be
 158     # a class variable if this and parseDiffTreeEntry were a part of a class.
 159     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160     while True:
 161         yield pattern
 162
 163 def parseDiffTreeEntry(entry):
 164     """Parses a single diff tree entry into its component elements.
 165
 166     See git-diff-tree(1) manpage for details about the format of the diff
 167     output. This method returns a dictionary with the following elements:
 168
 169     src_mode - The mode of the source file
 170     dst_mode - The mode of the destination file
 171     src_sha1 - The sha1 for the source file
 172     dst_sha1 - The sha1 fr the destination file
 173     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174     status_score - The score for the status (applicable for 'C' and 'R'
 175                    statuses). This is None if there is no score.
 176     src - The path for the source file.
 177     dst - The path for the destination file. This is only present for
 178           copy or renames. If it is not present, this is None.
 179
 180     If the pattern is not matched, None is returned."""
 181
 182     match = diffTreePattern().next().match(entry)
 183     if match:
 184         return {
 185             'src_mode': match.group(1),
 186             'dst_mode': match.group(2),
 187             'src_sha1': match.group(3),
 188             'dst_sha1': match.group(4),
 189             'status': match.group(5),
 190             'status_score': match.group(6),
 191             'src': match.group(7),
 192             'dst': match.group(10)
 193         }
 194     return None
 195
 196 def isModeExec(mode):
 197     # Returns True if the given git mode represents an executable file,
 198     # otherwise False.
 199     return mode[-3:] == "755"
 200
 201 def isModeExecChanged(src_mode, dst_mode):
 202     return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205     cmd = p4_build_cmd("-G %s" % (cmd))
 206     if verbose:
 207         sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209     # Use a temporary file to avoid deadlocks without
 210     # subprocess.communicate(), which would put another copy
 211     # of stdout into memory.
 212     stdin_file = None
 213     if stdin is not None:
 214         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215         stdin_file.write(stdin)
 216         stdin_file.flush()
 217         stdin_file.seek(0)
 218
 219     p4 = subprocess.Popen(cmd, shell=True,
 220                           stdin=stdin_file,
 221                           stdout=subprocess.PIPE)
 222
 223     result = []
 224     nchunks = nfiles = 0
 225     try:
 226         while True:
 227             entry = marshal.load(p4.stdout)
 228             result.append(entry)
 229             if entry.has_key("depotFile"):
 230                 nfiles += 1
 231             nchunks += 1
 232             if verbose:
 233                 sys.stderr.write("\r%s chunks (%s files)"
 234                                     % (nchunks, nfiles))
 235     except EOFError:
 236         pass
 237     if verbose:
 238         sys.stderr.write("\n")
 239     exitCode = p4.wait()
 240     if exitCode != 0:
 241         entry = {}
 242         entry["p4ExitCode"] = exitCode
 243         result.append(entry)
 244
 245     return result
 246
 247 def p4Cmd(cmd):
 248     list = p4CmdList(cmd)
 249     result = {}
 250     for entry in list:
 251         result.update(entry)
 252     return result
 253
 254 def p4Where(depotPath):
 255     if not depotPath.endswith("/"):
 256         depotPath += "/"
 257     depotPath = depotPath + "..."
 258     outputList = p4CmdList("where %s" % depotPath)
 259     output = None
 260     for entry in outputList:
 261         if "depotFile" in entry:
 262             if entry["depotFile"] == depotPath:
 263                 output = entry
 264                 break
 265         elif "data" in entry:
 266             data = entry.get("data")
 267             space = data.find(" ")
 268             if data[:space] == depotPath:
 269                 output = entry
 270                 break
 271     if output == None:
 272         return ""
 273     if output["code"] == "error":
 274         return ""
 275     clientPath = ""
 276     if "path" in output:
 277         clientPath = output.get("path")
 278     elif "data" in output:
 279         data = output.get("data")
 280         lastSpace = data.rfind(" ")
 281         clientPath = data[lastSpace + 1:]
 282
 283     if clientPath.endswith("..."):
 284         clientPath = clientPath[:-3]
 285     return clientPath
 286
 287 def currentGitBranch():
 288     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 289
 290 def isValidGitDir(path):
 291     if (os.path.exists(path + "/HEAD")
 292         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 293         return True
 294     return False
 295
 296 def parseRevision(ref):
 297     return read_pipe("git rev-parse %s" % ref).strip()
 298
 299 def extractLogMessageFromGitCommit(commit):
 300     logMessage = ""
 301
 302     ## fixme: title is first line of commit, not 1st paragraph.
 303     foundTitle = False
 304     for log in read_pipe_lines("git cat-file commit %s" % commit):
 305        if not foundTitle:
 306            if len(log) == 1:
 307                foundTitle = True
 308            continue
 309
 310        logMessage += log
 311     return logMessage
 312
 313 def extractSettingsGitLog(log):
 314     values = {}
 315     for line in log.split("\n"):
 316         line = line.strip()
 317         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 318         if not m:
 319             continue
 320
 321         assignments = m.group(1).split (':')
 322         for a in assignments:
 323             vals = a.split ('=')
 324             key = vals[0].strip()
 325             val = ('='.join (vals[1:])).strip()
 326             if val.endswith ('\"') and val.startswith('"'):
 327                 val = val[1:-1]
 328
 329             values[key] = val
 330
 331     paths = values.get("depot-paths")
 332     if not paths:
 333         paths = values.get("depot-path")
 334     if paths:
 335         values['depot-paths'] = paths.split(',')
 336     return values
 337
 338 def gitBranchExists(branch):
 339     proc = subprocess.Popen(["git", "rev-parse", branch],
 340                             stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 341     return proc.wait() == 0
 342
 343 _gitConfig = {}
 344 def gitConfig(key):
 345     if not _gitConfig.has_key(key):
 346         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 347     return _gitConfig[key]
 348
 349 def p4BranchesInGit(branchesAreInRemotes = True):
 350     branches = {}
 351
 352     cmdline = "git rev-parse --symbolic "
 353     if branchesAreInRemotes:
 354         cmdline += " --remotes"
 355     else:
 356         cmdline += " --branches"
 357
 358     for line in read_pipe_lines(cmdline):
 359         line = line.strip()
 360
 361         ## only import to p4/
 362         if not line.startswith('p4/') or line == "p4/HEAD":
 363             continue
 364         branch = line
 365
 366         # strip off p4
 367         branch = re.sub ("^p4/", "", line)
 368
 369         branches[branch] = parseRevision(line)
 370     return branches
 371
 372 def findUpstreamBranchPoint(head = "HEAD"):
 373     branches = p4BranchesInGit()
 374     # map from depot-path to branch name
 375     branchByDepotPath = {}
 376     for branch in branches.keys():
 377         tip = branches[branch]
 378         log = extractLogMessageFromGitCommit(tip)
 379         settings = extractSettingsGitLog(log)
 380         if settings.has_key("depot-paths"):
 381             paths = ",".join(settings["depot-paths"])
 382             branchByDepotPath[paths] = "remotes/p4/" + branch
 383
 384     settings = None
 385     parent = 0
 386     while parent < 65535:
 387         commit = head + "~%s" % parent
 388         log = extractLogMessageFromGitCommit(commit)
 389         settings = extractSettingsGitLog(log)
 390         if settings.has_key("depot-paths"):
 391             paths = ",".join(settings["depot-paths"])
 392             if branchByDepotPath.has_key(paths):
 393                 return [branchByDepotPath[paths], settings]
 394
 395         parent = parent + 1
 396
 397     return ["", settings]
 398
 399 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 400     if not silent:
 401         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 402                % localRefPrefix)
 403
 404     originPrefix = "origin/p4/"
 405
 406     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 407         line = line.strip()
 408         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 409             continue
 410
 411         headName = line[len(originPrefix):]
 412         remoteHead = localRefPrefix + headName
 413         originHead = line
 414
 415         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 416         if (not original.has_key('depot-paths')
 417             or not original.has_key('change')):
 418             continue
 419
 420         update = False
 421         if not gitBranchExists(remoteHead):
 422             if verbose:
 423                 print "creating %s" % remoteHead
 424             update = True
 425         else:
 426             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 427             if settings.has_key('change') > 0:
 428                 if settings['depot-paths'] == original['depot-paths']:
 429                     originP4Change = int(original['change'])
 430                     p4Change = int(settings['change'])
 431                     if originP4Change > p4Change:
 432                         print ("%s (%s) is newer than %s (%s). "
 433                                "Updating p4 branch from origin."
 434                                % (originHead, originP4Change,
 435                                   remoteHead, p4Change))
 436                         update = True
 437                 else:
 438                     print ("Ignoring: %s was imported from %s while "
 439                            "%s was imported from %s"
 440                            % (originHead, ','.join(original['depot-paths']),
 441                               remoteHead, ','.join(settings['depot-paths'])))
 442
 443         if update:
 444             system("git update-ref %s %s" % (remoteHead, originHead))
 445
 446 def originP4BranchesExist():
 447         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 448
 449 def p4ChangesForPaths(depotPaths, changeRange):
 450     assert depotPaths
 451     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 452                                                         for p in depotPaths]))
 453
 454     changes = {}
 455     for line in output:
 456         changeNum = int(line.split(" ")[1])
 457         changes[changeNum] = True
 458
 459     changelist = changes.keys()
 460     changelist.sort()
 461     return changelist
 462
 463 class Command:
 464     def __init__(self):
 465         self.usage = "usage: %prog [options]"
 466         self.needsGit = True
 467
 468 class P4Debug(Command):
 469     def __init__(self):
 470         Command.__init__(self)
 471         self.options = [
 472             optparse.make_option("--verbose", dest="verbose", action="store_true",
 473                                  default=False),
 474             ]
 475         self.description = "A tool to debug the output of p4 -G."
 476         self.needsGit = False
 477         self.verbose = False
 478
 479     def run(self, args):
 480         j = 0
 481         for output in p4CmdList(" ".join(args)):
 482             print 'Element: %d' % j
 483             j += 1
 484             print output
 485         return True
 486
 487 class P4RollBack(Command):
 488     def __init__(self):
 489         Command.__init__(self)
 490         self.options = [
 491             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 492             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 493         ]
 494         self.description = "A tool to debug the multi-branch import. Don't use :)"
 495         self.verbose = False
 496         self.rollbackLocalBranches = False
 497
 498     def run(self, args):
 499         if len(args) != 1:
 500             return False
 501         maxChange = int(args[0])
 502
 503         if "p4ExitCode" in p4Cmd("changes -m 1"):
 504             die("Problems executing p4")
 505
 506         if self.rollbackLocalBranches:
 507             refPrefix = "refs/heads/"
 508             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 509         else:
 510             refPrefix = "refs/remotes/"
 511             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 512
 513         for line in lines:
 514             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 515                 line = line.strip()
 516                 ref = refPrefix + line
 517                 log = extractLogMessageFromGitCommit(ref)
 518                 settings = extractSettingsGitLog(log)
 519
 520                 depotPaths = settings['depot-paths']
 521                 change = settings['change']
 522
 523                 changed = False
 524
 525                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 526                                                            for p in depotPaths]))) == 0:
 527                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 528                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 529                     continue
 530
 531                 while change and int(change) > maxChange:
 532                     changed = True
 533                     if self.verbose:
 534                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 535                     system("git update-ref %s \"%s^\"" % (ref, ref))
 536                     log = extractLogMessageFromGitCommit(ref)
 537                     settings =  extractSettingsGitLog(log)
 538
 539
 540                     depotPaths = settings['depot-paths']
 541                     change = settings['change']
 542
 543                 if changed:
 544                     print "%s rewound to %s" % (ref, change)
 545
 546         return True
 547
 548 class P4Submit(Command):
 549     def __init__(self):
 550         Command.__init__(self)
 551         self.options = [
 552                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 553                 optparse.make_option("--origin", dest="origin"),
 554                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 555         ]
 556         self.description = "Submit changes from git to the perforce depot."
 557         self.usage += " [name of git branch to submit into perforce depot]"
 558         self.interactive = True
 559         self.origin = ""
 560         self.detectRename = False
 561         self.verbose = False
 562         self.isWindows = (platform.system() == "Windows")
 563
 564     def check(self):
 565         if len(p4CmdList("opened ...")) > 0:
 566             die("You have files opened with perforce! Close them before starting the sync.")
 567
 568     # replaces everything between 'Description:' and the next P4 submit template field with the
 569     # commit message
 570     def prepareLogMessage(self, template, message):
 571         result = ""
 572
 573         inDescriptionSection = False
 574
 575         for line in template.split("\n"):
 576             if line.startswith("#"):
 577                 result += line + "\n"
 578                 continue
 579
 580             if inDescriptionSection:
 581                 if line.startswith("Files:"):
 582                     inDescriptionSection = False
 583                 else:
 584                     continue
 585             else:
 586                 if line.startswith("Description:"):
 587                     inDescriptionSection = True
 588                     line += "\n"
 589                     for messageLine in message.split("\n"):
 590                         line += "\t" + messageLine + "\n"
 591
 592             result += line + "\n"
 593
 594         return result
 595
 596     def prepareSubmitTemplate(self):
 597         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 598         template = ""
 599         inFilesSection = False
 600         for line in p4_read_pipe_lines("change -o"):
 601             if line.endswith("\r\n"):
 602                 line = line[:-2] + "\n"
 603             if inFilesSection:
 604                 if line.startswith("\t"):
 605                     # path starts and ends with a tab
 606                     path = line[1:]
 607                     lastTab = path.rfind("\t")
 608                     if lastTab != -1:
 609                         path = path[:lastTab]
 610                         if not path.startswith(self.depotPath):
 611                             continue
 612                 else:
 613                     inFilesSection = False
 614             else:
 615                 if line.startswith("Files:"):
 616                     inFilesSection = True
 617
 618             template += line
 619
 620         return template
 621
 622     def applyCommit(self, id):
 623         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 624         diffOpts = ("", "-M")[self.detectRename]
 625         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 626         filesToAdd = set()
 627         filesToDelete = set()
 628         editedFiles = set()
 629         filesToChangeExecBit = {}
 630         for line in diff:
 631             diff = parseDiffTreeEntry(line)
 632             modifier = diff['status']
 633             path = diff['src']
 634             if modifier == "M":
 635                 p4_system("edit \"%s\"" % path)
 636                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 637                     filesToChangeExecBit[path] = diff['dst_mode']
 638                 editedFiles.add(path)
 639             elif modifier == "A":
 640                 filesToAdd.add(path)
 641                 filesToChangeExecBit[path] = diff['dst_mode']
 642                 if path in filesToDelete:
 643                     filesToDelete.remove(path)
 644             elif modifier == "D":
 645                 filesToDelete.add(path)
 646                 if path in filesToAdd:
 647                     filesToAdd.remove(path)
 648             elif modifier == "R":
 649                 src, dest = diff['src'], diff['dst']
 650                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 651                 p4_system("edit \"%s\"" % (dest))
 652                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 653                     filesToChangeExecBit[dest] = diff['dst_mode']
 654                 os.unlink(dest)
 655                 editedFiles.add(dest)
 656                 filesToDelete.add(src)
 657             else:
 658                 die("unknown modifier %s for %s" % (modifier, path))
 659
 660         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 661         patchcmd = diffcmd + " | git apply "
 662         tryPatchCmd = patchcmd + "--check -"
 663         applyPatchCmd = patchcmd + "--check --apply -"
 664
 665         if os.system(tryPatchCmd) != 0:
 666             print "Unfortunately applying the change failed!"
 667             print "What do you want to do?"
 668             response = "x"
 669             while response != "s" and response != "a" and response != "w":
 670                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 671                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 672             if response == "s":
 673                 print "Skipping! Good luck with the next patches..."
 674                 for f in editedFiles:
 675                     p4_system("revert \"%s\"" % f)
 676                 for f in filesToAdd:
 677                     system("rm %s" %f)
 678                 return
 679             elif response == "a":
 680                 os.system(applyPatchCmd)
 681                 if len(filesToAdd) > 0:
 682                     print "You may also want to call p4 add on the following files:"
 683                     print " ".join(filesToAdd)
 684                 if len(filesToDelete):
 685                     print "The following files should be scheduled for deletion with p4 delete:"
 686                     print " ".join(filesToDelete)
 687                 die("Please resolve and submit the conflict manually and "
 688                     + "continue afterwards with git-p4 submit --continue")
 689             elif response == "w":
 690                 system(diffcmd + " > patch.txt")
 691                 print "Patch saved to patch.txt in %s !" % self.clientPath
 692                 die("Please resolve and submit the conflict manually and "
 693                     "continue afterwards with git-p4 submit --continue")
 694
 695         system(applyPatchCmd)
 696
 697         for f in filesToAdd:
 698             p4_system("add \"%s\"" % f)
 699         for f in filesToDelete:
 700             p4_system("revert \"%s\"" % f)
 701             p4_system("delete \"%s\"" % f)
 702
 703         # Set/clear executable bits
 704         for f in filesToChangeExecBit.keys():
 705             mode = filesToChangeExecBit[f]
 706             setP4ExecBit(f, mode)
 707
 708         logMessage = extractLogMessageFromGitCommit(id)
 709         logMessage = logMessage.strip()
 710
 711         template = self.prepareSubmitTemplate()
 712
 713         if self.interactive:
 714             submitTemplate = self.prepareLogMessage(template, logMessage)
 715             if os.environ.has_key("P4DIFF"):
 716                 del(os.environ["P4DIFF"])
 717             diff = p4_read_pipe("diff -du ...")
 718
 719             newdiff = ""
 720             for newFile in filesToAdd:
 721                 newdiff += "==== new file ====\n"
 722                 newdiff += "--- /dev/null\n"
 723                 newdiff += "+++ %s\n" % newFile
 724                 f = open(newFile, "r")
 725                 for line in f.readlines():
 726                     newdiff += "+" + line
 727                 f.close()
 728
 729             separatorLine = "######## everything below this line is just the diff #######\n"
 730
 731             [handle, fileName] = tempfile.mkstemp()
 732             tmpFile = os.fdopen(handle, "w+")
 733             if self.isWindows:
 734                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 735                 separatorLine = separatorLine.replace("\n", "\r\n")
 736                 newdiff = newdiff.replace("\n", "\r\n")
 737             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 738             tmpFile.close()
 739             mtime = os.stat(fileName).st_mtime
 740             defaultEditor = "vi"
 741             if platform.system() == "Windows":
 742                 defaultEditor = "notepad"
 743             if os.environ.has_key("P4EDITOR"):
 744                 editor = os.environ.get("P4EDITOR")
 745             else:
 746                 editor = os.environ.get("EDITOR", defaultEditor)
 747             system(editor + " " + fileName)
 748
 749             response = "y"
 750             if os.stat(fileName).st_mtime <= mtime:
 751                 response = "x"
 752                 while response != "y" and response != "n":
 753                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 754
 755             if response == "y":
 756                 tmpFile = open(fileName, "rb")
 757                 message = tmpFile.read()
 758                 tmpFile.close()
 759                 submitTemplate = message[:message.index(separatorLine)]
 760                 if self.isWindows:
 761                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 762                 p4_write_pipe("submit -i", submitTemplate)
 763             else:
 764                 for f in editedFiles:
 765                     p4_system("revert \"%s\"" % f)
 766                 for f in filesToAdd:
 767                     p4_system("revert \"%s\"" % f)
 768                     system("rm %s" %f)
 769
 770             os.remove(fileName)
 771         else:
 772             fileName = "submit.txt"
 773             file = open(fileName, "w+")
 774             file.write(self.prepareLogMessage(template, logMessage))
 775             file.close()
 776             print ("Perforce submit template written as %s. "
 777                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 778                    % (fileName, fileName))
 779
 780     def run(self, args):
 781         if len(args) == 0:
 782             self.master = currentGitBranch()
 783             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 784                 die("Detecting current git branch failed!")
 785         elif len(args) == 1:
 786             self.master = args[0]
 787         else:
 788             return False
 789
 790         allowSubmit = gitConfig("git-p4.allowSubmit")
 791         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 792             die("%s is not in git-p4.allowSubmit" % self.master)
 793
 794         [upstream, settings] = findUpstreamBranchPoint()
 795         self.depotPath = settings['depot-paths'][0]
 796         if len(self.origin) == 0:
 797             self.origin = upstream
 798
 799         if self.verbose:
 800             print "Origin branch is " + self.origin
 801
 802         if len(self.depotPath) == 0:
 803             print "Internal error: cannot locate perforce depot path from existing branches"
 804             sys.exit(128)
 805
 806         self.clientPath = p4Where(self.depotPath)
 807
 808         if len(self.clientPath) == 0:
 809             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 810             sys.exit(128)
 811
 812         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 813         self.oldWorkingDirectory = os.getcwd()
 814
 815         chdir(self.clientPath)
 816         print "Syncronizing p4 checkout..."
 817         p4_system("sync ...")
 818
 819         self.check()
 820
 821         commits = []
 822         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 823             commits.append(line.strip())
 824         commits.reverse()
 825
 826         while len(commits) > 0:
 827             commit = commits[0]
 828             commits = commits[1:]
 829             self.applyCommit(commit)
 830             if not self.interactive:
 831                 break
 832
 833         if len(commits) == 0:
 834             print "All changes applied!"
 835             chdir(self.oldWorkingDirectory)
 836
 837             sync = P4Sync()
 838             sync.run([])
 839
 840             rebase = P4Rebase()
 841             rebase.rebase()
 842
 843         return True
 844
 845 class P4Sync(Command):
 846     def __init__(self):
 847         Command.__init__(self)
 848         self.options = [
 849                 optparse.make_option("--branch", dest="branch"),
 850                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 851                 optparse.make_option("--changesfile", dest="changesFile"),
 852                 optparse.make_option("--silent", dest="silent", action="store_true"),
 853                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 854                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 855                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 856                                      help="Import into refs/heads/ , not refs/remotes"),
 857                 optparse.make_option("--max-changes", dest="maxChanges"),
 858                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 859                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 860                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 861                                      help="Only sync files that are included in the Perforce Client Spec")
 862         ]
 863         self.description = """Imports from Perforce into a git repository.\n
 864     example:
 865     //depot/my/project/ -- to import the current head
 866     //depot/my/project/@all -- to import everything
 867     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 868
 869     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 870
 871         self.usage += " //depot/path[@revRange]"
 872         self.silent = False
 873         self.createdBranches = Set()
 874         self.committedChanges = Set()
 875         self.branch = ""
 876         self.detectBranches = False
 877         self.detectLabels = False
 878         self.changesFile = ""
 879         self.syncWithOrigin = True
 880         self.verbose = False
 881         self.importIntoRemotes = True
 882         self.maxChanges = ""
 883         self.isWindows = (platform.system() == "Windows")
 884         self.keepRepoPath = False
 885         self.depotPaths = None
 886         self.p4BranchesInGit = []
 887         self.cloneExclude = []
 888         self.useClientSpec = False
 889         self.clientSpecDirs = []
 890
 891         if gitConfig("git-p4.syncFromOrigin") == "false":
 892             self.syncWithOrigin = False
 893
 894     def extractFilesFromCommit(self, commit):
 895         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 896                              for path in self.cloneExclude]
 897         files = []
 898         fnum = 0
 899         while commit.has_key("depotFile%s" % fnum):
 900             path =  commit["depotFile%s" % fnum]
 901
 902             if [p for p in self.cloneExclude
 903                 if path.startswith (p)]:
 904                 found = False
 905             else:
 906                 found = [p for p in self.depotPaths
 907                          if path.startswith (p)]
 908             if not found:
 909                 fnum = fnum + 1
 910                 continue
 911
 912             file = {}
 913             file["path"] = path
 914             file["rev"] = commit["rev%s" % fnum]
 915             file["action"] = commit["action%s" % fnum]
 916             file["type"] = commit["type%s" % fnum]
 917             files.append(file)
 918             fnum = fnum + 1
 919         return files
 920
 921     def stripRepoPath(self, path, prefixes):
 922         if self.keepRepoPath:
 923             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 924
 925         for p in prefixes:
 926             if path.startswith(p):
 927                 path = path[len(p):]
 928
 929         return path
 930
 931     def splitFilesIntoBranches(self, commit):
 932         branches = {}
 933         fnum = 0
 934         while commit.has_key("depotFile%s" % fnum):
 935             path =  commit["depotFile%s" % fnum]
 936             found = [p for p in self.depotPaths
 937                      if path.startswith (p)]
 938             if not found:
 939                 fnum = fnum + 1
 940                 continue
 941
 942             file = {}
 943             file["path"] = path
 944             file["rev"] = commit["rev%s" % fnum]
 945             file["action"] = commit["action%s" % fnum]
 946             file["type"] = commit["type%s" % fnum]
 947             fnum = fnum + 1
 948
 949             relPath = self.stripRepoPath(path, self.depotPaths)
 950
 951             for branch in self.knownBranches.keys():
 952
 953                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 954                 if relPath.startswith(branch + "/"):
 955                     if branch not in branches:
 956                         branches[branch] = []
 957                     branches[branch].append(file)
 958                     break
 959
 960         return branches
 961
 962     ## Should move this out, doesn't use SELF.
 963     def readP4Files(self, files):
 964         filesForCommit = []
 965         filesToRead = []
 966
 967         for f in files:
 968             includeFile = True
 969             for val in self.clientSpecDirs:
 970                 if f['path'].startswith(val[0]):
 971                     if val[1] <= 0:
 972                         includeFile = False
 973                     break
 974
 975             if includeFile:
 976                 filesForCommit.append(f)
 977                 if f['action'] not in ('delete', 'purge'):
 978                     filesToRead.append(f)
 979
 980         filedata = []
 981         if len(filesToRead) > 0:
 982             filedata = p4CmdList('-x - print',
 983                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 984                                                   for f in filesToRead]),
 985                                  stdin_mode='w+')
 986
 987             if "p4ExitCode" in filedata[0]:
 988                 die("Problems executing p4. Error: [%d]."
 989                     % (filedata[0]['p4ExitCode']))
 990
 991         j = 0
 992         contents = {}
 993         while j < len(filedata):
 994             stat = filedata[j]
 995             j += 1
 996             data = []
 997             text = ''
 998             # Append data every 8192 chunks to 1) ensure decent performance
 999             # by not making too many string concatenations and 2) avoid
1000             # excessive memory usage by purging "data" often enough. p4
1001             # sends 4k chunks, so we should not use more than 32 MiB of
1002             # additional memory while rebuilding the file data.
1003             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
1004                 data.append(filedata[j]['data'])
1005                 del filedata[j]['data']
1006                 if len(data) >= 8192:
1007                     text += ''.join(data)
1008                     data = []
1009                 j += 1
1010             text += ''.join(data)
1011             data = None
1012
1013             if not stat.has_key('depotFile'):
1014                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
1015                 continue
1016
1017             if len(text) >= gitConfig('core.bigFileThreshold'):
1018                 pass
1019             elif stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1020                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
1021             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1022                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text)
1023
1024             contents[stat['depotFile']] = text
1025
1026         for f in filesForCommit:
1027             path = f['path']
1028             if contents.has_key(path):
1029                 f['data'] = contents[path]
1030
1031         return filesForCommit
1032
1033     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1034         epoch = details["time"]
1035         author = details["user"]
1036
1037         if self.verbose:
1038             print "commit into %s" % branch
1039
1040         # start with reading files; if that fails, we should not
1041         # create a commit.
1042         new_files = []
1043         for f in files:
1044             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1045                 new_files.append (f)
1046             else:
1047                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1048         files = self.readP4Files(new_files)
1049
1050         self.gitStream.write("commit %s\n" % branch)
1051 #        gitStream.write("mark :%s\n" % details["change"])
1052         self.committedChanges.add(int(details["change"]))
1053         committer = ""
1054         if author not in self.users:
1055             self.getUserMapFromPerforceServer()
1056         if author in self.users:
1057             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1058         else:
1059             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1060
1061         self.gitStream.write("committer %s\n" % committer)
1062
1063         self.gitStream.write("data <<EOT\n")
1064         self.gitStream.write(details["desc"])
1065         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1066                              % (','.join (branchPrefixes), details["change"]))
1067         if len(details['options']) > 0:
1068             self.gitStream.write(": options = %s" % details['options'])
1069         self.gitStream.write("]\nEOT\n\n")
1070
1071         if len(parent) > 0:
1072             if self.verbose:
1073                 print "parent %s" % parent
1074             self.gitStream.write("from %s\n" % parent)
1075
1076         for file in files:
1077             if file["type"] == "apple":
1078                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1079                 continue
1080
1081             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1082             if file["action"] in ("delete", "purge"):
1083                 self.gitStream.write("D %s\n" % relPath)
1084             else:
1085                 data = file['data']
1086
1087                 mode = "644"
1088                 if isP4Exec(file["type"]):
1089                     mode = "755"
1090                 elif file["type"] == "symlink":
1091                     mode = "120000"
1092                     # p4 print on a symlink contains "target\n", so strip it off
1093                     data = data[:-1]
1094
1095                 if self.isWindows and file["type"].endswith("text") \
1096                    and len(data) < gitConfig('core.bigFileThreshold'):
1097                     data = data.replace("\r\n", "\n")
1098
1099                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1100                 self.gitStream.write("data %s\n" % len(data))
1101                 self.gitStream.write(data)
1102                 self.gitStream.write("\n")
1103
1104         self.gitStream.write("\n")
1105
1106         change = int(details["change"])
1107
1108         if self.labels.has_key(change):
1109             label = self.labels[change]
1110             labelDetails = label[0]
1111             labelRevisions = label[1]
1112             if self.verbose:
1113                 print "Change %s is labelled %s" % (change, labelDetails)
1114
1115             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1116                                                     for p in branchPrefixes]))
1117
1118             if len(files) == len(labelRevisions):
1119
1120                 cleanedFiles = {}
1121                 for info in files:
1122                     if info["action"] in ("delete", "purge"):
1123                         continue
1124                     cleanedFiles[info["depotFile"]] = info["rev"]
1125
1126                 if cleanedFiles == labelRevisions:
1127                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1128                     self.gitStream.write("from %s\n" % branch)
1129
1130                     owner = labelDetails["Owner"]
1131                     tagger = ""
1132                     if author in self.users:
1133                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1134                     else:
1135                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1136                     self.gitStream.write("tagger %s\n" % tagger)
1137                     self.gitStream.write("data <<EOT\n")
1138                     self.gitStream.write(labelDetails["Description"])
1139                     self.gitStream.write("EOT\n\n")
1140
1141                 else:
1142                     if not self.silent:
1143                         print ("Tag %s does not match with change %s: files do not match."
1144                                % (labelDetails["label"], change))
1145
1146             else:
1147                 if not self.silent:
1148                     print ("Tag %s does not match with change %s: file count is different."
1149                            % (labelDetails["label"], change))
1150
1151     def getUserCacheFilename(self):
1152         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1153         return home + "/.gitp4-usercache.txt"
1154
1155     def getUserMapFromPerforceServer(self):
1156         if self.userMapFromPerforceServer:
1157             return
1158         self.users = {}
1159
1160         for output in p4CmdList("users"):
1161             if not output.has_key("User"):
1162                 continue
1163             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1164
1165
1166         s = ''
1167         for (key, val) in self.users.items():
1168             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1169
1170         open(self.getUserCacheFilename(), "wb").write(s)
1171         self.userMapFromPerforceServer = True
1172
1173     def loadUserMapFromCache(self):
1174         self.users = {}
1175         self.userMapFromPerforceServer = False
1176         try:
1177             cache = open(self.getUserCacheFilename(), "rb")
1178             lines = cache.readlines()
1179             cache.close()
1180             for line in lines:
1181                 entry = line.strip().split("\t")
1182                 self.users[entry[0]] = entry[1]
1183         except IOError:
1184             self.getUserMapFromPerforceServer()
1185
1186     def getLabels(self):
1187         self.labels = {}
1188
1189         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1190         if len(l) > 0 and not self.silent:
1191             print "Finding files belonging to labels in %s" % `self.depotPaths`
1192
1193         for output in l:
1194             label = output["label"]
1195             revisions = {}
1196             newestChange = 0
1197             if self.verbose:
1198                 print "Querying files for label %s" % label
1199             for file in p4CmdList("files "
1200                                   +  ' '.join (["%s...@%s" % (p, label)
1201                                                 for p in self.depotPaths])):
1202                 revisions[file["depotFile"]] = file["rev"]
1203                 change = int(file["change"])
1204                 if change > newestChange:
1205                     newestChange = change
1206
1207             self.labels[newestChange] = [output, revisions]
1208
1209         if self.verbose:
1210             print "Label changes: %s" % self.labels.keys()
1211
1212     def guessProjectName(self):
1213         for p in self.depotPaths:
1214             if p.endswith("/"):
1215                 p = p[:-1]
1216             p = p[p.strip().rfind("/") + 1:]
1217             if not p.endswith("/"):
1218                p += "/"
1219             return p
1220
1221     def getBranchMapping(self):
1222         lostAndFoundBranches = set()
1223
1224         for info in p4CmdList("branches"):
1225             details = p4Cmd("branch -o %s" % info["branch"])
1226             viewIdx = 0
1227             while details.has_key("View%s" % viewIdx):
1228                 paths = details["View%s" % viewIdx].split(" ")
1229                 viewIdx = viewIdx + 1
1230                 # require standard //depot/foo/... //depot/bar/... mapping
1231                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1232                     continue
1233                 source = paths[0]
1234                 destination = paths[1]
1235                 ## HACK
1236                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1237                     source = source[len(self.depotPaths[0]):-4]
1238                     destination = destination[len(self.depotPaths[0]):-4]
1239
1240                     if destination in self.knownBranches:
1241                         if not self.silent:
1242                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1243                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1244                         continue
1245
1246                     self.knownBranches[destination] = source
1247
1248                     lostAndFoundBranches.discard(destination)
1249
1250                     if source not in self.knownBranches:
1251                         lostAndFoundBranches.add(source)
1252
1253
1254         for branch in lostAndFoundBranches:
1255             self.knownBranches[branch] = branch
1256
1257     def getBranchMappingFromGitBranches(self):
1258         branches = p4BranchesInGit(self.importIntoRemotes)
1259         for branch in branches.keys():
1260             if branch == "master":
1261                 branch = "main"
1262             else:
1263                 branch = branch[len(self.projectName):]
1264             self.knownBranches[branch] = branch
1265
1266     def listExistingP4GitBranches(self):
1267         # branches holds mapping from name to commit
1268         branches = p4BranchesInGit(self.importIntoRemotes)
1269         self.p4BranchesInGit = branches.keys()
1270         for branch in branches.keys():
1271             self.initialParents[self.refPrefix + branch] = branches[branch]
1272
1273     def updateOptionDict(self, d):
1274         option_keys = {}
1275         if self.keepRepoPath:
1276             option_keys['keepRepoPath'] = 1
1277
1278         d["options"] = ' '.join(sorted(option_keys.keys()))
1279
1280     def readOptions(self, d):
1281         self.keepRepoPath = (d.has_key('options')
1282                              and ('keepRepoPath' in d['options']))
1283
1284     def gitRefForBranch(self, branch):
1285         if branch == "main":
1286             return self.refPrefix + "master"
1287
1288         if len(branch) <= 0:
1289             return branch
1290
1291         return self.refPrefix + self.projectName + branch
1292
1293     def gitCommitByP4Change(self, ref, change):
1294         if self.verbose:
1295             print "looking in ref " + ref + " for change %s using bisect..." % change
1296
1297         earliestCommit = ""
1298         latestCommit = parseRevision(ref)
1299
1300         while True:
1301             if self.verbose:
1302                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1303             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1304             if len(next) == 0:
1305                 if self.verbose:
1306                     print "argh"
1307                 return ""
1308             log = extractLogMessageFromGitCommit(next)
1309             settings = extractSettingsGitLog(log)
1310             currentChange = int(settings['change'])
1311             if self.verbose:
1312                 print "current change %s" % currentChange
1313
1314             if currentChange == change:
1315                 if self.verbose:
1316                     print "found %s" % next
1317                 return next
1318
1319             if currentChange < change:
1320                 earliestCommit = "^%s" % next
1321             else:
1322                 latestCommit = "%s" % next
1323
1324         return ""
1325
1326     def importNewBranch(self, branch, maxChange):
1327         # make fast-import flush all changes to disk and update the refs using the checkpoint
1328         # command so that we can try to find the branch parent in the git history
1329         self.gitStream.write("checkpoint\n\n")
1330         self.gitStream.flush()
1331         branchPrefix = self.depotPaths[0] + branch + "/"
1332         range = "@1,%s" % maxChange
1333         #print "prefix" + branchPrefix
1334         changes = p4ChangesForPaths([branchPrefix], range)
1335         if len(changes) <= 0:
1336             return False
1337         firstChange = changes[0]
1338         #print "first change in branch: %s" % firstChange
1339         sourceBranch = self.knownBranches[branch]
1340         sourceDepotPath = self.depotPaths[0] + sourceBranch
1341         sourceRef = self.gitRefForBranch(sourceBranch)
1342         #print "source " + sourceBranch
1343
1344         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1345         #print "branch parent: %s" % branchParentChange
1346         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1347         if len(gitParent) > 0:
1348             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1349             #print "parent git commit: %s" % gitParent
1350
1351         self.importChanges(changes)
1352         return True
1353
1354     def importChanges(self, changes):
1355         cnt = 1
1356         for change in changes:
1357             description = p4Cmd("describe %s" % change)
1358             self.updateOptionDict(description)
1359
1360             if not self.silent:
1361                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1362                 sys.stdout.flush()
1363             cnt = cnt + 1
1364
1365             try:
1366                 if self.detectBranches:
1367                     branches = self.splitFilesIntoBranches(description)
1368                     for branch in branches.keys():
1369                         ## HACK  --hwn
1370                         branchPrefix = self.depotPaths[0] + branch + "/"
1371
1372                         parent = ""
1373
1374                         filesForCommit = branches[branch]
1375
1376                         if self.verbose:
1377                             print "branch is %s" % branch
1378
1379                         self.updatedBranches.add(branch)
1380
1381                         if branch not in self.createdBranches:
1382                             self.createdBranches.add(branch)
1383                             parent = self.knownBranches[branch]
1384                             if parent == branch:
1385                                 parent = ""
1386                             else:
1387                                 fullBranch = self.projectName + branch
1388                                 if fullBranch not in self.p4BranchesInGit:
1389                                     if not self.silent:
1390                                         print("\n    Importing new branch %s" % fullBranch)
1391                                     if self.importNewBranch(branch, change - 1):
1392                                         parent = ""
1393                                         self.p4BranchesInGit.append(fullBranch)
1394                                     if not self.silent:
1395                                         print("\n    Resuming with change %s" % change)
1396
1397                                 if self.verbose:
1398                                     print "parent determined through known branches: %s" % parent
1399
1400                         branch = self.gitRefForBranch(branch)
1401                         parent = self.gitRefForBranch(parent)
1402
1403                         if self.verbose:
1404                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1405
1406                         if len(parent) == 0 and branch in self.initialParents:
1407                             parent = self.initialParents[branch]
1408                             del self.initialParents[branch]
1409
1410                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1411                 else:
1412                     files = self.extractFilesFromCommit(description)
1413                     self.commit(description, files, self.branch, self.depotPaths,
1414                                 self.initialParent)
1415                     self.initialParent = ""
1416             except IOError:
1417                 print self.gitError.read()
1418                 sys.exit(1)
1419
1420     def importHeadRevision(self, revision):
1421         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1422
1423         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1424         details["desc"] = ("Initial import of %s from the state at revision %s"
1425                            % (' '.join(self.depotPaths), revision))
1426         details["change"] = revision
1427         newestRevision = 0
1428
1429         fileCnt = 0
1430         for info in p4CmdList("files "
1431                               +  ' '.join(["%s...%s"
1432                                            % (p, revision)
1433                                            for p in self.depotPaths])):
1434
1435             if info['code'] == 'error':
1436                 sys.stderr.write("p4 returned an error: %s\n"
1437                                  % info['data'])
1438                 sys.exit(1)
1439
1440
1441             change = int(info["change"])
1442             if change > newestRevision:
1443                 newestRevision = change
1444
1445             if info["action"] in ("delete", "purge"):
1446                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1447                 #fileCnt = fileCnt + 1
1448                 continue
1449
1450             for prop in ["depotFile", "rev", "action", "type" ]:
1451                 details["%s%s" % (prop, fileCnt)] = info[prop]
1452
1453             fileCnt = fileCnt + 1
1454
1455         details["change"] = newestRevision
1456         self.updateOptionDict(details)
1457         try:
1458             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1459         except IOError:
1460             print "IO error with git fast-import. Is your git version recent enough?"
1461             print self.gitError.read()
1462
1463
1464     def getClientSpec(self):
1465         specList = p4CmdList( "client -o" )
1466         temp = {}
1467         for entry in specList:
1468             for k,v in entry.iteritems():
1469                 if k.startswith("View"):
1470                     if v.startswith('"'):
1471                         start = 1
1472                     else:
1473                         start = 0
1474                     index = v.find("...")
1475                     v = v[start:index]
1476                     if v.startswith("-"):
1477                         v = v[1:]
1478                         temp[v] = -len(v)
1479                     else:
1480                         temp[v] = len(v)
1481         self.clientSpecDirs = temp.items()
1482         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1483
1484     def run(self, args):
1485         self.depotPaths = []
1486         self.changeRange = ""
1487         self.initialParent = ""
1488         self.previousDepotPaths = []
1489
1490         # map from branch depot path to parent branch
1491         self.knownBranches = {}
1492         self.initialParents = {}
1493         self.hasOrigin = originP4BranchesExist()
1494         if not self.syncWithOrigin:
1495             self.hasOrigin = False
1496
1497         if self.importIntoRemotes:
1498             self.refPrefix = "refs/remotes/p4/"
1499         else:
1500             self.refPrefix = "refs/heads/p4/"
1501
1502         if self.syncWithOrigin and self.hasOrigin:
1503             if not self.silent:
1504                 print "Syncing with origin first by calling git fetch origin"
1505             system("git fetch origin")
1506
1507         if len(self.branch) == 0:
1508             self.branch = self.refPrefix + "master"
1509             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1510                 system("git update-ref %s refs/heads/p4" % self.branch)
1511                 system("git branch -D p4")
1512             # create it /after/ importing, when master exists
1513             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1514                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1515
1516         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1517             self.getClientSpec()
1518
1519         # TODO: should always look at previous commits,
1520         # merge with previous imports, if possible.
1521         if args == []:
1522             if self.hasOrigin:
1523                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1524             self.listExistingP4GitBranches()
1525
1526             if len(self.p4BranchesInGit) > 1:
1527                 if not self.silent:
1528                     print "Importing from/into multiple branches"
1529                 self.detectBranches = True
1530
1531             if self.verbose:
1532                 print "branches: %s" % self.p4BranchesInGit
1533
1534             p4Change = 0
1535             for branch in self.p4BranchesInGit:
1536                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1537
1538                 settings = extractSettingsGitLog(logMsg)
1539
1540                 self.readOptions(settings)
1541                 if (settings.has_key('depot-paths')
1542                     and settings.has_key ('change')):
1543                     change = int(settings['change']) + 1
1544                     p4Change = max(p4Change, change)
1545
1546                     depotPaths = sorted(settings['depot-paths'])
1547                     if self.previousDepotPaths == []:
1548                         self.previousDepotPaths = depotPaths
1549                     else:
1550                         paths = []
1551                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1552                             for i in range(0, min(len(cur), len(prev))):
1553                                 if cur[i] <> prev[i]:
1554                                     i = i - 1
1555                                     break
1556
1557                             paths.append (cur[:i + 1])
1558
1559                         self.previousDepotPaths = paths
1560
1561             if p4Change > 0:
1562                 self.depotPaths = sorted(self.previousDepotPaths)
1563                 self.changeRange = "@%s,#head" % p4Change
1564                 if not self.detectBranches:
1565                     self.initialParent = parseRevision(self.branch)
1566                 if not self.silent and not self.detectBranches:
1567                     print "Performing incremental import into %s git branch" % self.branch
1568
1569         if not self.branch.startswith("refs/"):
1570             self.branch = "refs/heads/" + self.branch
1571
1572         if len(args) == 0 and self.depotPaths:
1573             if not self.silent:
1574                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1575         else:
1576             if self.depotPaths and self.depotPaths != args:
1577                 print ("previous import used depot path %s and now %s was specified. "
1578                        "This doesn't work!" % (' '.join (self.depotPaths),
1579                                                ' '.join (args)))
1580                 sys.exit(1)
1581
1582             self.depotPaths = sorted(args)
1583
1584         revision = ""
1585         self.users = {}
1586
1587         newPaths = []
1588         for p in self.depotPaths:
1589             if p.find("@") != -1:
1590                 atIdx = p.index("@")
1591                 self.changeRange = p[atIdx:]
1592                 if self.changeRange == "@all":
1593                     self.changeRange = ""
1594                 elif ',' not in self.changeRange:
1595                     revision = self.changeRange
1596                     self.changeRange = ""
1597                 p = p[:atIdx]
1598             elif p.find("#") != -1:
1599                 hashIdx = p.index("#")
1600                 revision = p[hashIdx:]
1601                 p = p[:hashIdx]
1602             elif self.previousDepotPaths == []:
1603                 revision = "#head"
1604
1605             p = re.sub ("\.\.\.$", "", p)
1606             if not p.endswith("/"):
1607                 p += "/"
1608
1609             newPaths.append(p)
1610
1611         self.depotPaths = newPaths
1612
1613
1614         self.loadUserMapFromCache()
1615         self.labels = {}
1616         if self.detectLabels:
1617             self.getLabels()
1618
1619         if self.detectBranches:
1620             ## FIXME - what's a P4 projectName ?
1621             self.projectName = self.guessProjectName()
1622
1623             if self.hasOrigin:
1624                 self.getBranchMappingFromGitBranches()
1625             else:
1626                 self.getBranchMapping()
1627             if self.verbose:
1628                 print "p4-git branches: %s" % self.p4BranchesInGit
1629                 print "initial parents: %s" % self.initialParents
1630             for b in self.p4BranchesInGit:
1631                 if b != "master":
1632
1633                     ## FIXME
1634                     b = b[len(self.projectName):]
1635                 self.createdBranches.add(b)
1636
1637         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1638
1639         importProcess = subprocess.Popen(["git", "fast-import"],
1640                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1641                                          stderr=subprocess.PIPE)
1642         self.gitOutput = importProcess.stdout
1643         self.gitStream = importProcess.stdin
1644         self.gitError = importProcess.stderr
1645
1646         if revision:
1647             self.importHeadRevision(revision)
1648         else:
1649             changes = []
1650
1651             if len(self.changesFile) > 0:
1652                 output = open(self.changesFile).readlines()
1653                 changeSet = Set()
1654                 for line in output:
1655                     changeSet.add(int(line))
1656
1657                 for change in changeSet:
1658                     changes.append(change)
1659
1660                 changes.sort()
1661             else:
1662                 if self.verbose:
1663                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1664                                                               self.changeRange)
1665                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1666
1667                 if len(self.maxChanges) > 0:
1668                     changes = changes[:min(int(self.maxChanges), len(changes))]
1669
1670             if len(changes) == 0:
1671                 if not self.silent:
1672                     print "No changes to import!"
1673                 return True
1674
1675             if not self.silent and not self.detectBranches:
1676                 print "Import destination: %s" % self.branch
1677
1678             self.updatedBranches = set()
1679
1680             self.importChanges(changes)
1681
1682             if not self.silent:
1683                 print ""
1684                 if len(self.updatedBranches) > 0:
1685                     sys.stdout.write("Updated branches: ")
1686                     for b in self.updatedBranches:
1687                         sys.stdout.write("%s " % b)
1688                     sys.stdout.write("\n")
1689
1690         self.gitStream.close()
1691         if importProcess.wait() != 0:
1692             die("fast-import failed: %s" % self.gitError.read())
1693         self.gitOutput.close()
1694         self.gitError.close()
1695
1696         return True
1697
1698 class P4Rebase(Command):
1699     def __init__(self):
1700         Command.__init__(self)
1701         self.options = [ ]
1702         self.description = ("Fetches the latest revision from perforce and "
1703                             + "rebases the current work (branch) against it")
1704         self.verbose = False
1705
1706     def run(self, args):
1707         sync = P4Sync()
1708         sync.run([])
1709
1710         return self.rebase()
1711
1712     def rebase(self):
1713         if os.system("git update-index --refresh") != 0:
1714             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.")
1715         if len(read_pipe("git diff-index HEAD --")) > 0:
1716             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.")
1717
1718         [upstream, settings] = findUpstreamBranchPoint()
1719         if len(upstream) == 0:
1720             die("Cannot find upstream branchpoint for rebase")
1721
1722         # the branchpoint may be p4/foo~3, so strip off the parent
1723         upstream = re.sub("~[0-9]+$", "", upstream)
1724
1725         print "Rebasing the current branch onto %s" % upstream
1726         oldHead = read_pipe("git rev-parse HEAD").strip()
1727         system("git rebase %s" % upstream)
1728         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1729         return True
1730
1731 class P4Clone(P4Sync):
1732     def __init__(self):
1733         P4Sync.__init__(self)
1734         self.description = "Creates a new git repository and imports from Perforce into it"
1735         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1736         self.options += [
1737             optparse.make_option("--destination", dest="cloneDestination",
1738                                  action='store', default=None,
1739                                  help="where to leave result of the clone"),
1740             optparse.make_option("-/", dest="cloneExclude",
1741                                  action="append", type="string",
1742                                  help="exclude depot path")
1743         ]
1744         self.cloneDestination = None
1745         self.needsGit = False
1746
1747     # This is required for the "append" cloneExclude action
1748     def ensure_value(self, attr, value):
1749         if not hasattr(self, attr) or getattr(self, attr) is None:
1750             setattr(self, attr, value)
1751         return getattr(self, attr)
1752
1753     def defaultDestination(self, args):
1754         ## TODO: use common prefix of args?
1755         depotPath = args[0]
1756         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1757         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1758         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1759         depotDir = re.sub(r"/$", "", depotDir)
1760         return os.path.split(depotDir)[1]
1761
1762     def run(self, args):
1763         if len(args) < 1:
1764             return False
1765
1766         if self.keepRepoPath and not self.cloneDestination:
1767             sys.stderr.write("Must specify destination for --keep-path\n")
1768             sys.exit(1)
1769
1770         depotPaths = args
1771
1772         if not self.cloneDestination and len(depotPaths) > 1:
1773             self.cloneDestination = depotPaths[-1]
1774             depotPaths = depotPaths[:-1]
1775
1776         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1777         for p in depotPaths:
1778             if not p.startswith("//"):
1779                 return False
1780
1781         if not self.cloneDestination:
1782             self.cloneDestination = self.defaultDestination(args)
1783
1784         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1785         if not os.path.exists(self.cloneDestination):
1786             os.makedirs(self.cloneDestination)
1787         chdir(self.cloneDestination)
1788         system("git init")
1789         self.gitdir = os.getcwd() + "/.git"
1790         if not P4Sync.run(self, depotPaths):
1791             return False
1792         if self.branch != "master":
1793             if self.importIntoRemotes:
1794                 masterbranch = "refs/remotes/p4/master"
1795             else:
1796                 masterbranch = "refs/heads/p4/master"
1797             if gitBranchExists(masterbranch):
1798                 system("git branch master %s" % masterbranch)
1799                 system("git checkout -f")
1800             else:
1801                 print "Could not detect main branch. No checkout/master branch created."
1802
1803         return True
1804
1805 class P4Branches(Command):
1806     def __init__(self):
1807         Command.__init__(self)
1808         self.options = [ ]
1809         self.description = ("Shows the git branches that hold imports and their "
1810                             + "corresponding perforce depot paths")
1811         self.verbose = False
1812
1813     def run(self, args):
1814         if originP4BranchesExist():
1815             createOrUpdateBranchesFromOrigin()
1816
1817         cmdline = "git rev-parse --symbolic "
1818         cmdline += " --remotes"
1819
1820         for line in read_pipe_lines(cmdline):
1821             line = line.strip()
1822
1823             if not line.startswith('p4/') or line == "p4/HEAD":
1824                 continue
1825             branch = line
1826
1827             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1828             settings = extractSettingsGitLog(log)
1829
1830             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1831         return True
1832
1833 class HelpFormatter(optparse.IndentedHelpFormatter):
1834     def __init__(self):
1835         optparse.IndentedHelpFormatter.__init__(self)
1836
1837     def format_description(self, description):
1838         if description:
1839             return description + "\n"
1840         else:
1841             return ""
1842
1843 def printUsage(commands):
1844     print "usage: %s <command> [options]" % sys.argv[0]
1845     print ""
1846     print "valid commands: %s" % ", ".join(commands)
1847     print ""
1848     print "Try %s <command> --help for command specific help." % sys.argv[0]
1849     print ""
1850
1851 commands = {
1852     "debug" : P4Debug,
1853     "submit" : P4Submit,
1854     "commit" : P4Submit,
1855     "sync" : P4Sync,
1856     "rebase" : P4Rebase,
1857     "clone" : P4Clone,
1858     "rollback" : P4RollBack,
1859     "branches" : P4Branches
1860 }
1861
1862
1863 def main():
1864     if len(sys.argv[1:]) == 0:
1865         printUsage(commands.keys())
1866         sys.exit(2)
1867
1868     cmd = ""
1869     cmdName = sys.argv[1]
1870     try:
1871         klass = commands[cmdName]
1872         cmd = klass()
1873     except KeyError:
1874         print "unknown command %s" % cmdName
1875         print ""
1876         printUsage(commands.keys())
1877         sys.exit(2)
1878
1879     options = cmd.options
1880     cmd.gitdir = os.environ.get("GIT_DIR", None)
1881
1882     args = sys.argv[2:]
1883
1884     if len(options) > 0:
1885         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1886
1887         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1888                                        options,
1889                                        description = cmd.description,
1890                                        formatter = HelpFormatter())
1891
1892         (cmd, args) = parser.parse_args(sys.argv[2:], cmd)
1893     global verbose
1894     verbose = cmd.verbose
1895     if cmd.needsGit:
1896         if cmd.gitdir == None:
1897             cmd.gitdir = os.path.abspath(".git")
1898             if not isValidGitDir(cmd.gitdir):
1899                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1900                 if os.path.exists(cmd.gitdir):
1901                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1902                     if len(cdup) > 0:
1903                         chdir(cdup)
1904
1905         if not isValidGitDir(cmd.gitdir):
1906             if isValidGitDir(cmd.gitdir + "/.git"):
1907                 cmd.gitdir += "/.git"
1908             else:
1909                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1910
1911         os.environ["GIT_DIR"] = cmd.gitdir
1912
1913     if not cmd.run(args):
1914         parser.print_help()
1915
1916
1917 if __name__ == '__main__':
1918     main()