contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key):
 337     if not _gitConfig.has_key(key):
 338         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339     return _gitConfig[key]
 340
 341 def p4BranchesInGit(branchesAreInRemotes = True):
 342     branches = {}
 343
 344     cmdline = "git rev-parse --symbolic "
 345     if branchesAreInRemotes:
 346         cmdline += " --remotes"
 347     else:
 348         cmdline += " --branches"
 349
 350     for line in read_pipe_lines(cmdline):
 351         line = line.strip()
 352
 353         ## only import to p4/
 354         if not line.startswith('p4/') or line == "p4/HEAD":
 355             continue
 356         branch = line
 357
 358         # strip off p4
 359         branch = re.sub ("^p4/", "", line)
 360
 361         branches[branch] = parseRevision(line)
 362     return branches
 363
 364 def findUpstreamBranchPoint(head = "HEAD"):
 365     branches = p4BranchesInGit()
 366     # map from depot-path to branch name
 367     branchByDepotPath = {}
 368     for branch in branches.keys():
 369         tip = branches[branch]
 370         log = extractLogMessageFromGitCommit(tip)
 371         settings = extractSettingsGitLog(log)
 372         if settings.has_key("depot-paths"):
 373             paths = ",".join(settings["depot-paths"])
 374             branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376     settings = None
 377     parent = 0
 378     while parent < 65535:
 379         commit = head + "~%s" % parent
 380         log = extractLogMessageFromGitCommit(commit)
 381         settings = extractSettingsGitLog(log)
 382         if settings.has_key("depot-paths"):
 383             paths = ",".join(settings["depot-paths"])
 384             if branchByDepotPath.has_key(paths):
 385                 return [branchByDepotPath[paths], settings]
 386
 387         parent = parent + 1
 388
 389     return ["", settings]
 390
 391 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392     if not silent:
 393         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394                % localRefPrefix)
 395
 396     originPrefix = "origin/p4/"
 397
 398     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399         line = line.strip()
 400         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401             continue
 402
 403         headName = line[len(originPrefix):]
 404         remoteHead = localRefPrefix + headName
 405         originHead = line
 406
 407         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408         if (not original.has_key('depot-paths')
 409             or not original.has_key('change')):
 410             continue
 411
 412         update = False
 413         if not gitBranchExists(remoteHead):
 414             if verbose:
 415                 print "creating %s" % remoteHead
 416             update = True
 417         else:
 418             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419             if settings.has_key('change') > 0:
 420                 if settings['depot-paths'] == original['depot-paths']:
 421                     originP4Change = int(original['change'])
 422                     p4Change = int(settings['change'])
 423                     if originP4Change > p4Change:
 424                         print ("%s (%s) is newer than %s (%s). "
 425                                "Updating p4 branch from origin."
 426                                % (originHead, originP4Change,
 427                                   remoteHead, p4Change))
 428                         update = True
 429                 else:
 430                     print ("Ignoring: %s was imported from %s while "
 431                            "%s was imported from %s"
 432                            % (originHead, ','.join(original['depot-paths']),
 433                               remoteHead, ','.join(settings['depot-paths'])))
 434
 435         if update:
 436             system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438 def originP4BranchesExist():
 439         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441 def p4ChangesForPaths(depotPaths, changeRange):
 442     assert depotPaths
 443     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                         for p in depotPaths]))
 445
 446     changes = {}
 447     for line in output:
 448         changeNum = int(line.split(" ")[1])
 449         changes[changeNum] = True
 450
 451     changelist = changes.keys()
 452     changelist.sort()
 453     return changelist
 454
 455 class Command:
 456     def __init__(self):
 457         self.usage = "usage: %prog [options]"
 458         self.needsGit = True
 459
 460 class P4Debug(Command):
 461     def __init__(self):
 462         Command.__init__(self)
 463         self.options = [
 464             optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                  default=False),
 466             ]
 467         self.description = "A tool to debug the output of p4 -G."
 468         self.needsGit = False
 469         self.verbose = False
 470
 471     def run(self, args):
 472         j = 0
 473         for output in p4CmdList(" ".join(args)):
 474             print 'Element: %d' % j
 475             j += 1
 476             print output
 477         return True
 478
 479 class P4RollBack(Command):
 480     def __init__(self):
 481         Command.__init__(self)
 482         self.options = [
 483             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485         ]
 486         self.description = "A tool to debug the multi-branch import. Don't use :)"
 487         self.verbose = False
 488         self.rollbackLocalBranches = False
 489
 490     def run(self, args):
 491         if len(args) != 1:
 492             return False
 493         maxChange = int(args[0])
 494
 495         if "p4ExitCode" in p4Cmd("changes -m 1"):
 496             die("Problems executing p4");
 497
 498         if self.rollbackLocalBranches:
 499             refPrefix = "refs/heads/"
 500             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501         else:
 502             refPrefix = "refs/remotes/"
 503             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505         for line in lines:
 506             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                 line = line.strip()
 508                 ref = refPrefix + line
 509                 log = extractLogMessageFromGitCommit(ref)
 510                 settings = extractSettingsGitLog(log)
 511
 512                 depotPaths = settings['depot-paths']
 513                 change = settings['change']
 514
 515                 changed = False
 516
 517                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                            for p in depotPaths]))) == 0:
 519                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                     continue
 522
 523                 while change and int(change) > maxChange:
 524                     changed = True
 525                     if self.verbose:
 526                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                     system("git update-ref %s \"%s^\"" % (ref, ref))
 528                     log = extractLogMessageFromGitCommit(ref)
 529                     settings =  extractSettingsGitLog(log)
 530
 531
 532                     depotPaths = settings['depot-paths']
 533                     change = settings['change']
 534
 535                 if changed:
 536                     print "%s rewound to %s" % (ref, change)
 537
 538         return True
 539
 540 class P4Submit(Command):
 541     def __init__(self):
 542         Command.__init__(self)
 543         self.options = [
 544                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                 optparse.make_option("--origin", dest="origin"),
 546                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 547         ]
 548         self.description = "Submit changes from git to the perforce depot."
 549         self.usage += " [name of git branch to submit into perforce depot]"
 550         self.interactive = True
 551         self.origin = ""
 552         self.detectRenames = False
 553         self.verbose = False
 554         self.isWindows = (platform.system() == "Windows")
 555
 556     def check(self):
 557         if len(p4CmdList("opened ...")) > 0:
 558             die("You have files opened with perforce! Close them before starting the sync.")
 559
 560     # replaces everything between 'Description:' and the next P4 submit template field with the
 561     # commit message
 562     def prepareLogMessage(self, template, message):
 563         result = ""
 564
 565         inDescriptionSection = False
 566
 567         for line in template.split("\n"):
 568             if line.startswith("#"):
 569                 result += line + "\n"
 570                 continue
 571
 572             if inDescriptionSection:
 573                 if line.startswith("Files:"):
 574                     inDescriptionSection = False
 575                 else:
 576                     continue
 577             else:
 578                 if line.startswith("Description:"):
 579                     inDescriptionSection = True
 580                     line += "\n"
 581                     for messageLine in message.split("\n"):
 582                         line += "\t" + messageLine + "\n"
 583
 584             result += line + "\n"
 585
 586         return result
 587
 588     def prepareSubmitTemplate(self):
 589         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590         template = ""
 591         inFilesSection = False
 592         for line in p4_read_pipe_lines("change -o"):
 593             if line.endswith("\r\n"):
 594                 line = line[:-2] + "\n"
 595             if inFilesSection:
 596                 if line.startswith("\t"):
 597                     # path starts and ends with a tab
 598                     path = line[1:]
 599                     lastTab = path.rfind("\t")
 600                     if lastTab != -1:
 601                         path = path[:lastTab]
 602                         if not path.startswith(self.depotPath):
 603                             continue
 604                 else:
 605                     inFilesSection = False
 606             else:
 607                 if line.startswith("Files:"):
 608                     inFilesSection = True
 609
 610             template += line
 611
 612         return template
 613
 614     def applyCommit(self, id):
 615         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616
 617         if not self.detectRenames:
 618             # If not explicitly set check the config variable
 619             self.detectRenames = gitConfig("git-p4.detectRenames").lower() == "true"
 620
 621         if self.detectRenames:
 622             diffOpts = "-M"
 623         else:
 624             diffOpts = ""
 625
 626         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 627         filesToAdd = set()
 628         filesToDelete = set()
 629         editedFiles = set()
 630         filesToChangeExecBit = {}
 631         for line in diff:
 632             diff = parseDiffTreeEntry(line)
 633             modifier = diff['status']
 634             path = diff['src']
 635             if modifier == "M":
 636                 p4_system("edit \"%s\"" % path)
 637                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 638                     filesToChangeExecBit[path] = diff['dst_mode']
 639                 editedFiles.add(path)
 640             elif modifier == "A":
 641                 filesToAdd.add(path)
 642                 filesToChangeExecBit[path] = diff['dst_mode']
 643                 if path in filesToDelete:
 644                     filesToDelete.remove(path)
 645             elif modifier == "D":
 646                 filesToDelete.add(path)
 647                 if path in filesToAdd:
 648                     filesToAdd.remove(path)
 649             elif modifier == "R":
 650                 src, dest = diff['src'], diff['dst']
 651                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 652                 if diff['src_sha1'] != diff['dst_sha1']:
 653                     p4_system("edit \"%s\"" % (dest))
 654                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 655                     p4_system("edit \"%s\"" % (dest))
 656                     filesToChangeExecBit[dest] = diff['dst_mode']
 657                 os.unlink(dest)
 658                 editedFiles.add(dest)
 659                 filesToDelete.add(src)
 660             else:
 661                 die("unknown modifier %s for %s" % (modifier, path))
 662
 663         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 664         patchcmd = diffcmd + " | git apply "
 665         tryPatchCmd = patchcmd + "--check -"
 666         applyPatchCmd = patchcmd + "--check --apply -"
 667
 668         if os.system(tryPatchCmd) != 0:
 669             print "Unfortunately applying the change failed!"
 670             print "What do you want to do?"
 671             response = "x"
 672             while response != "s" and response != "a" and response != "w":
 673                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 674                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 675             if response == "s":
 676                 print "Skipping! Good luck with the next patches..."
 677                 for f in editedFiles:
 678                     p4_system("revert \"%s\"" % f);
 679                 for f in filesToAdd:
 680                     system("rm %s" %f)
 681                 return
 682             elif response == "a":
 683                 os.system(applyPatchCmd)
 684                 if len(filesToAdd) > 0:
 685                     print "You may also want to call p4 add on the following files:"
 686                     print " ".join(filesToAdd)
 687                 if len(filesToDelete):
 688                     print "The following files should be scheduled for deletion with p4 delete:"
 689                     print " ".join(filesToDelete)
 690                 die("Please resolve and submit the conflict manually and "
 691                     + "continue afterwards with git-p4 submit --continue")
 692             elif response == "w":
 693                 system(diffcmd + " > patch.txt")
 694                 print "Patch saved to patch.txt in %s !" % self.clientPath
 695                 die("Please resolve and submit the conflict manually and "
 696                     "continue afterwards with git-p4 submit --continue")
 697
 698         system(applyPatchCmd)
 699
 700         for f in filesToAdd:
 701             p4_system("add \"%s\"" % f)
 702         for f in filesToDelete:
 703             p4_system("revert \"%s\"" % f)
 704             p4_system("delete \"%s\"" % f)
 705
 706         # Set/clear executable bits
 707         for f in filesToChangeExecBit.keys():
 708             mode = filesToChangeExecBit[f]
 709             setP4ExecBit(f, mode)
 710
 711         logMessage = extractLogMessageFromGitCommit(id)
 712         logMessage = logMessage.strip()
 713
 714         template = self.prepareSubmitTemplate()
 715
 716         if self.interactive:
 717             submitTemplate = self.prepareLogMessage(template, logMessage)
 718             if os.environ.has_key("P4DIFF"):
 719                 del(os.environ["P4DIFF"])
 720             diff = ""
 721             for editedFile in editedFiles:
 722                 diff += p4_read_pipe("diff -du %r" % editedFile)
 723
 724             newdiff = ""
 725             for newFile in filesToAdd:
 726                 newdiff += "==== new file ====\n"
 727                 newdiff += "--- /dev/null\n"
 728                 newdiff += "+++ %s\n" % newFile
 729                 f = open(newFile, "r")
 730                 for line in f.readlines():
 731                     newdiff += "+" + line
 732                 f.close()
 733
 734             separatorLine = "######## everything below this line is just the diff #######\n"
 735
 736             [handle, fileName] = tempfile.mkstemp()
 737             tmpFile = os.fdopen(handle, "w+")
 738             if self.isWindows:
 739                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 740                 separatorLine = separatorLine.replace("\n", "\r\n")
 741                 newdiff = newdiff.replace("\n", "\r\n")
 742             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 743             tmpFile.close()
 744             mtime = os.stat(fileName).st_mtime
 745             if os.environ.has_key("P4EDITOR"):
 746                 editor = os.environ.get("P4EDITOR")
 747             else:
 748                 editor = read_pipe("git var GIT_EDITOR").strip()
 749             system(editor + " " + fileName)
 750
 751             response = "y"
 752             if os.stat(fileName).st_mtime <= mtime:
 753                 response = "x"
 754                 while response != "y" and response != "n":
 755                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 756
 757             if response == "y":
 758                 tmpFile = open(fileName, "rb")
 759                 message = tmpFile.read()
 760                 tmpFile.close()
 761                 submitTemplate = message[:message.index(separatorLine)]
 762                 if self.isWindows:
 763                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 764                 p4_write_pipe("submit -i", submitTemplate)
 765             else:
 766                 for f in editedFiles:
 767                     p4_system("revert \"%s\"" % f);
 768                 for f in filesToAdd:
 769                     p4_system("revert \"%s\"" % f);
 770                     system("rm %s" %f)
 771
 772             os.remove(fileName)
 773         else:
 774             fileName = "submit.txt"
 775             file = open(fileName, "w+")
 776             file.write(self.prepareLogMessage(template, logMessage))
 777             file.close()
 778             print ("Perforce submit template written as %s. "
 779                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 780                    % (fileName, fileName))
 781
 782     def run(self, args):
 783         if len(args) == 0:
 784             self.master = currentGitBranch()
 785             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 786                 die("Detecting current git branch failed!")
 787         elif len(args) == 1:
 788             self.master = args[0]
 789         else:
 790             return False
 791
 792         allowSubmit = gitConfig("git-p4.allowSubmit")
 793         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 794             die("%s is not in git-p4.allowSubmit" % self.master)
 795
 796         [upstream, settings] = findUpstreamBranchPoint()
 797         self.depotPath = settings['depot-paths'][0]
 798         if len(self.origin) == 0:
 799             self.origin = upstream
 800
 801         if self.verbose:
 802             print "Origin branch is " + self.origin
 803
 804         if len(self.depotPath) == 0:
 805             print "Internal error: cannot locate perforce depot path from existing branches"
 806             sys.exit(128)
 807
 808         self.clientPath = p4Where(self.depotPath)
 809
 810         if len(self.clientPath) == 0:
 811             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 812             sys.exit(128)
 813
 814         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 815         self.oldWorkingDirectory = os.getcwd()
 816
 817         chdir(self.clientPath)
 818         print "Synchronizing p4 checkout..."
 819         p4_system("sync ...")
 820
 821         self.check()
 822
 823         commits = []
 824         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 825             commits.append(line.strip())
 826         commits.reverse()
 827
 828         while len(commits) > 0:
 829             commit = commits[0]
 830             commits = commits[1:]
 831             self.applyCommit(commit)
 832             if not self.interactive:
 833                 break
 834
 835         if len(commits) == 0:
 836             print "All changes applied!"
 837             chdir(self.oldWorkingDirectory)
 838
 839             sync = P4Sync()
 840             sync.run([])
 841
 842             rebase = P4Rebase()
 843             rebase.rebase()
 844
 845         return True
 846
 847 class P4Sync(Command):
 848     def __init__(self):
 849         Command.__init__(self)
 850         self.options = [
 851                 optparse.make_option("--branch", dest="branch"),
 852                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 853                 optparse.make_option("--changesfile", dest="changesFile"),
 854                 optparse.make_option("--silent", dest="silent", action="store_true"),
 855                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 856                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 857                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 858                                      help="Import into refs/heads/ , not refs/remotes"),
 859                 optparse.make_option("--max-changes", dest="maxChanges"),
 860                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 861                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 862                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 863                                      help="Only sync files that are included in the Perforce Client Spec")
 864         ]
 865         self.description = """Imports from Perforce into a git repository.\n
 866     example:
 867     //depot/my/project/ -- to import the current head
 868     //depot/my/project/@all -- to import everything
 869     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 870
 871     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 872
 873         self.usage += " //depot/path[@revRange]"
 874         self.silent = False
 875         self.createdBranches = set()
 876         self.committedChanges = set()
 877         self.branch = ""
 878         self.detectBranches = False
 879         self.detectLabels = False
 880         self.changesFile = ""
 881         self.syncWithOrigin = True
 882         self.verbose = False
 883         self.importIntoRemotes = True
 884         self.maxChanges = ""
 885         self.isWindows = (platform.system() == "Windows")
 886         self.keepRepoPath = False
 887         self.depotPaths = None
 888         self.p4BranchesInGit = []
 889         self.cloneExclude = []
 890         self.useClientSpec = False
 891         self.clientSpecDirs = []
 892
 893         if gitConfig("git-p4.syncFromOrigin") == "false":
 894             self.syncWithOrigin = False
 895
 896     def extractFilesFromCommit(self, commit):
 897         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 898                              for path in self.cloneExclude]
 899         files = []
 900         fnum = 0
 901         while commit.has_key("depotFile%s" % fnum):
 902             path =  commit["depotFile%s" % fnum]
 903
 904             if [p for p in self.cloneExclude
 905                 if path.startswith (p)]:
 906                 found = False
 907             else:
 908                 found = [p for p in self.depotPaths
 909                          if path.startswith (p)]
 910             if not found:
 911                 fnum = fnum + 1
 912                 continue
 913
 914             file = {}
 915             file["path"] = path
 916             file["rev"] = commit["rev%s" % fnum]
 917             file["action"] = commit["action%s" % fnum]
 918             file["type"] = commit["type%s" % fnum]
 919             files.append(file)
 920             fnum = fnum + 1
 921         return files
 922
 923     def stripRepoPath(self, path, prefixes):
 924         if self.keepRepoPath:
 925             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 926
 927         for p in prefixes:
 928             if path.startswith(p):
 929                 path = path[len(p):]
 930
 931         return path
 932
 933     def splitFilesIntoBranches(self, commit):
 934         branches = {}
 935         fnum = 0
 936         while commit.has_key("depotFile%s" % fnum):
 937             path =  commit["depotFile%s" % fnum]
 938             found = [p for p in self.depotPaths
 939                      if path.startswith (p)]
 940             if not found:
 941                 fnum = fnum + 1
 942                 continue
 943
 944             file = {}
 945             file["path"] = path
 946             file["rev"] = commit["rev%s" % fnum]
 947             file["action"] = commit["action%s" % fnum]
 948             file["type"] = commit["type%s" % fnum]
 949             fnum = fnum + 1
 950
 951             relPath = self.stripRepoPath(path, self.depotPaths)
 952
 953             for branch in self.knownBranches.keys():
 954
 955                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 956                 if relPath.startswith(branch + "/"):
 957                     if branch not in branches:
 958                         branches[branch] = []
 959                     branches[branch].append(file)
 960                     break
 961
 962         return branches
 963
 964     # output one file from the P4 stream
 965     # - helper for streamP4Files
 966
 967     def streamOneP4File(self, file, contents):
 968         if file["type"] == "apple":
 969             print "\nfile %s is a strange apple file that forks. Ignoring" % \
 970                 file['depotFile']
 971             return
 972
 973         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 974         if verbose:
 975             sys.stderr.write("%s\n" % relPath)
 976
 977         mode = "644"
 978         if isP4Exec(file["type"]):
 979             mode = "755"
 980         elif file["type"] == "symlink":
 981             mode = "120000"
 982             # p4 print on a symlink contains "target\n", so strip it off
 983             data = ''.join(contents)
 984             contents = [data[:-1]]
 985
 986         if self.isWindows and file["type"].endswith("text"):
 987             mangled = []
 988             for data in contents:
 989                 data = data.replace("\r\n", "\n")
 990                 mangled.append(data)
 991             contents = mangled
 992
 993         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 994             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 995         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 996             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 997
 998         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 999
1000         # total length...
1001         length = 0
1002         for d in contents:
1003             length = length + len(d)
1004
1005         self.gitStream.write("data %d\n" % length)
1006         for d in contents:
1007             self.gitStream.write(d)
1008         self.gitStream.write("\n")
1009
1010     def streamOneP4Deletion(self, file):
1011         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1012         if verbose:
1013             sys.stderr.write("delete %s\n" % relPath)
1014         self.gitStream.write("D %s\n" % relPath)
1015
1016     # handle another chunk of streaming data
1017     def streamP4FilesCb(self, marshalled):
1018
1019         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1020             # start of a new file - output the old one first
1021             self.streamOneP4File(self.stream_file, self.stream_contents)
1022             self.stream_file = {}
1023             self.stream_contents = []
1024             self.stream_have_file_info = False
1025
1026         # pick up the new file information... for the
1027         # 'data' field we need to append to our array
1028         for k in marshalled.keys():
1029             if k == 'data':
1030                 self.stream_contents.append(marshalled['data'])
1031             else:
1032                 self.stream_file[k] = marshalled[k]
1033
1034         self.stream_have_file_info = True
1035
1036     # Stream directly from "p4 files" into "git fast-import"
1037     def streamP4Files(self, files):
1038         filesForCommit = []
1039         filesToRead = []
1040         filesToDelete = []
1041
1042         for f in files:
1043             includeFile = True
1044             for val in self.clientSpecDirs:
1045                 if f['path'].startswith(val[0]):
1046                     if val[1] <= 0:
1047                         includeFile = False
1048                     break
1049
1050             if includeFile:
1051                 filesForCommit.append(f)
1052                 if f['action'] not in ('delete', 'move/delete', 'purge'):
1053                     filesToRead.append(f)
1054                 else:
1055                     filesToDelete.append(f)
1056
1057         # deleted files...
1058         for f in filesToDelete:
1059             self.streamOneP4Deletion(f)
1060
1061         if len(filesToRead) > 0:
1062             self.stream_file = {}
1063             self.stream_contents = []
1064             self.stream_have_file_info = False
1065
1066             # curry self argument
1067             def streamP4FilesCbSelf(entry):
1068                 self.streamP4FilesCb(entry)
1069
1070             p4CmdList("-x - print",
1071                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1072                                                   for f in filesToRead]),
1073                 cb=streamP4FilesCbSelf)
1074
1075             # do the last chunk
1076             if self.stream_file.has_key('depotFile'):
1077                 self.streamOneP4File(self.stream_file, self.stream_contents)
1078
1079     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1080         epoch = details["time"]
1081         author = details["user"]
1082         self.branchPrefixes = branchPrefixes
1083
1084         if self.verbose:
1085             print "commit into %s" % branch
1086
1087         # start with reading files; if that fails, we should not
1088         # create a commit.
1089         new_files = []
1090         for f in files:
1091             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1092                 new_files.append (f)
1093             else:
1094                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1095
1096         self.gitStream.write("commit %s\n" % branch)
1097 #        gitStream.write("mark :%s\n" % details["change"])
1098         self.committedChanges.add(int(details["change"]))
1099         committer = ""
1100         if author not in self.users:
1101             self.getUserMapFromPerforceServer()
1102         if author in self.users:
1103             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1104         else:
1105             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1106
1107         self.gitStream.write("committer %s\n" % committer)
1108
1109         self.gitStream.write("data <<EOT\n")
1110         self.gitStream.write(details["desc"])
1111         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1112                              % (','.join (branchPrefixes), details["change"]))
1113         if len(details['options']) > 0:
1114             self.gitStream.write(": options = %s" % details['options'])
1115         self.gitStream.write("]\nEOT\n\n")
1116
1117         if len(parent) > 0:
1118             if self.verbose:
1119                 print "parent %s" % parent
1120             self.gitStream.write("from %s\n" % parent)
1121
1122         self.streamP4Files(new_files)
1123         self.gitStream.write("\n")
1124
1125         change = int(details["change"])
1126
1127         if self.labels.has_key(change):
1128             label = self.labels[change]
1129             labelDetails = label[0]
1130             labelRevisions = label[1]
1131             if self.verbose:
1132                 print "Change %s is labelled %s" % (change, labelDetails)
1133
1134             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1135                                                     for p in branchPrefixes]))
1136
1137             if len(files) == len(labelRevisions):
1138
1139                 cleanedFiles = {}
1140                 for info in files:
1141                     if info["action"] in ("delete", "purge"):
1142                         continue
1143                     cleanedFiles[info["depotFile"]] = info["rev"]
1144
1145                 if cleanedFiles == labelRevisions:
1146                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1147                     self.gitStream.write("from %s\n" % branch)
1148
1149                     owner = labelDetails["Owner"]
1150                     tagger = ""
1151                     if author in self.users:
1152                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1153                     else:
1154                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1155                     self.gitStream.write("tagger %s\n" % tagger)
1156                     self.gitStream.write("data <<EOT\n")
1157                     self.gitStream.write(labelDetails["Description"])
1158                     self.gitStream.write("EOT\n\n")
1159
1160                 else:
1161                     if not self.silent:
1162                         print ("Tag %s does not match with change %s: files do not match."
1163                                % (labelDetails["label"], change))
1164
1165             else:
1166                 if not self.silent:
1167                     print ("Tag %s does not match with change %s: file count is different."
1168                            % (labelDetails["label"], change))
1169
1170     def getUserCacheFilename(self):
1171         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1172         return home + "/.gitp4-usercache.txt"
1173
1174     def getUserMapFromPerforceServer(self):
1175         if self.userMapFromPerforceServer:
1176             return
1177         self.users = {}
1178
1179         for output in p4CmdList("users"):
1180             if not output.has_key("User"):
1181                 continue
1182             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1183
1184
1185         s = ''
1186         for (key, val) in self.users.items():
1187             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1188
1189         open(self.getUserCacheFilename(), "wb").write(s)
1190         self.userMapFromPerforceServer = True
1191
1192     def loadUserMapFromCache(self):
1193         self.users = {}
1194         self.userMapFromPerforceServer = False
1195         try:
1196             cache = open(self.getUserCacheFilename(), "rb")
1197             lines = cache.readlines()
1198             cache.close()
1199             for line in lines:
1200                 entry = line.strip().split("\t")
1201                 self.users[entry[0]] = entry[1]
1202         except IOError:
1203             self.getUserMapFromPerforceServer()
1204
1205     def getLabels(self):
1206         self.labels = {}
1207
1208         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1209         if len(l) > 0 and not self.silent:
1210             print "Finding files belonging to labels in %s" % `self.depotPaths`
1211
1212         for output in l:
1213             label = output["label"]
1214             revisions = {}
1215             newestChange = 0
1216             if self.verbose:
1217                 print "Querying files for label %s" % label
1218             for file in p4CmdList("files "
1219                                   +  ' '.join (["%s...@%s" % (p, label)
1220                                                 for p in self.depotPaths])):
1221                 revisions[file["depotFile"]] = file["rev"]
1222                 change = int(file["change"])
1223                 if change > newestChange:
1224                     newestChange = change
1225
1226             self.labels[newestChange] = [output, revisions]
1227
1228         if self.verbose:
1229             print "Label changes: %s" % self.labels.keys()
1230
1231     def guessProjectName(self):
1232         for p in self.depotPaths:
1233             if p.endswith("/"):
1234                 p = p[:-1]
1235             p = p[p.strip().rfind("/") + 1:]
1236             if not p.endswith("/"):
1237                p += "/"
1238             return p
1239
1240     def getBranchMapping(self):
1241         lostAndFoundBranches = set()
1242
1243         for info in p4CmdList("branches"):
1244             details = p4Cmd("branch -o %s" % info["branch"])
1245             viewIdx = 0
1246             while details.has_key("View%s" % viewIdx):
1247                 paths = details["View%s" % viewIdx].split(" ")
1248                 viewIdx = viewIdx + 1
1249                 # require standard //depot/foo/... //depot/bar/... mapping
1250                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1251                     continue
1252                 source = paths[0]
1253                 destination = paths[1]
1254                 ## HACK
1255                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1256                     source = source[len(self.depotPaths[0]):-4]
1257                     destination = destination[len(self.depotPaths[0]):-4]
1258
1259                     if destination in self.knownBranches:
1260                         if not self.silent:
1261                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1262                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1263                         continue
1264
1265                     self.knownBranches[destination] = source
1266
1267                     lostAndFoundBranches.discard(destination)
1268
1269                     if source not in self.knownBranches:
1270                         lostAndFoundBranches.add(source)
1271
1272
1273         for branch in lostAndFoundBranches:
1274             self.knownBranches[branch] = branch
1275
1276     def getBranchMappingFromGitBranches(self):
1277         branches = p4BranchesInGit(self.importIntoRemotes)
1278         for branch in branches.keys():
1279             if branch == "master":
1280                 branch = "main"
1281             else:
1282                 branch = branch[len(self.projectName):]
1283             self.knownBranches[branch] = branch
1284
1285     def listExistingP4GitBranches(self):
1286         # branches holds mapping from name to commit
1287         branches = p4BranchesInGit(self.importIntoRemotes)
1288         self.p4BranchesInGit = branches.keys()
1289         for branch in branches.keys():
1290             self.initialParents[self.refPrefix + branch] = branches[branch]
1291
1292     def updateOptionDict(self, d):
1293         option_keys = {}
1294         if self.keepRepoPath:
1295             option_keys['keepRepoPath'] = 1
1296
1297         d["options"] = ' '.join(sorted(option_keys.keys()))
1298
1299     def readOptions(self, d):
1300         self.keepRepoPath = (d.has_key('options')
1301                              and ('keepRepoPath' in d['options']))
1302
1303     def gitRefForBranch(self, branch):
1304         if branch == "main":
1305             return self.refPrefix + "master"
1306
1307         if len(branch) <= 0:
1308             return branch
1309
1310         return self.refPrefix + self.projectName + branch
1311
1312     def gitCommitByP4Change(self, ref, change):
1313         if self.verbose:
1314             print "looking in ref " + ref + " for change %s using bisect..." % change
1315
1316         earliestCommit = ""
1317         latestCommit = parseRevision(ref)
1318
1319         while True:
1320             if self.verbose:
1321                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1322             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1323             if len(next) == 0:
1324                 if self.verbose:
1325                     print "argh"
1326                 return ""
1327             log = extractLogMessageFromGitCommit(next)
1328             settings = extractSettingsGitLog(log)
1329             currentChange = int(settings['change'])
1330             if self.verbose:
1331                 print "current change %s" % currentChange
1332
1333             if currentChange == change:
1334                 if self.verbose:
1335                     print "found %s" % next
1336                 return next
1337
1338             if currentChange < change:
1339                 earliestCommit = "^%s" % next
1340             else:
1341                 latestCommit = "%s" % next
1342
1343         return ""
1344
1345     def importNewBranch(self, branch, maxChange):
1346         # make fast-import flush all changes to disk and update the refs using the checkpoint
1347         # command so that we can try to find the branch parent in the git history
1348         self.gitStream.write("checkpoint\n\n");
1349         self.gitStream.flush();
1350         branchPrefix = self.depotPaths[0] + branch + "/"
1351         range = "@1,%s" % maxChange
1352         #print "prefix" + branchPrefix
1353         changes = p4ChangesForPaths([branchPrefix], range)
1354         if len(changes) <= 0:
1355             return False
1356         firstChange = changes[0]
1357         #print "first change in branch: %s" % firstChange
1358         sourceBranch = self.knownBranches[branch]
1359         sourceDepotPath = self.depotPaths[0] + sourceBranch
1360         sourceRef = self.gitRefForBranch(sourceBranch)
1361         #print "source " + sourceBranch
1362
1363         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1364         #print "branch parent: %s" % branchParentChange
1365         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1366         if len(gitParent) > 0:
1367             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1368             #print "parent git commit: %s" % gitParent
1369
1370         self.importChanges(changes)
1371         return True
1372
1373     def importChanges(self, changes):
1374         cnt = 1
1375         for change in changes:
1376             description = p4Cmd("describe %s" % change)
1377             self.updateOptionDict(description)
1378
1379             if not self.silent:
1380                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1381                 sys.stdout.flush()
1382             cnt = cnt + 1
1383
1384             try:
1385                 if self.detectBranches:
1386                     branches = self.splitFilesIntoBranches(description)
1387                     for branch in branches.keys():
1388                         ## HACK  --hwn
1389                         branchPrefix = self.depotPaths[0] + branch + "/"
1390
1391                         parent = ""
1392
1393                         filesForCommit = branches[branch]
1394
1395                         if self.verbose:
1396                             print "branch is %s" % branch
1397
1398                         self.updatedBranches.add(branch)
1399
1400                         if branch not in self.createdBranches:
1401                             self.createdBranches.add(branch)
1402                             parent = self.knownBranches[branch]
1403                             if parent == branch:
1404                                 parent = ""
1405                             else:
1406                                 fullBranch = self.projectName + branch
1407                                 if fullBranch not in self.p4BranchesInGit:
1408                                     if not self.silent:
1409                                         print("\n    Importing new branch %s" % fullBranch);
1410                                     if self.importNewBranch(branch, change - 1):
1411                                         parent = ""
1412                                         self.p4BranchesInGit.append(fullBranch)
1413                                     if not self.silent:
1414                                         print("\n    Resuming with change %s" % change);
1415
1416                                 if self.verbose:
1417                                     print "parent determined through known branches: %s" % parent
1418
1419                         branch = self.gitRefForBranch(branch)
1420                         parent = self.gitRefForBranch(parent)
1421
1422                         if self.verbose:
1423                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1424
1425                         if len(parent) == 0 and branch in self.initialParents:
1426                             parent = self.initialParents[branch]
1427                             del self.initialParents[branch]
1428
1429                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1430                 else:
1431                     files = self.extractFilesFromCommit(description)
1432                     self.commit(description, files, self.branch, self.depotPaths,
1433                                 self.initialParent)
1434                     self.initialParent = ""
1435             except IOError:
1436                 print self.gitError.read()
1437                 sys.exit(1)
1438
1439     def importHeadRevision(self, revision):
1440         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1441
1442         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1443         details["desc"] = ("Initial import of %s from the state at revision %s"
1444                            % (' '.join(self.depotPaths), revision))
1445         details["change"] = revision
1446         newestRevision = 0
1447
1448         fileCnt = 0
1449         for info in p4CmdList("files "
1450                               +  ' '.join(["%s...%s"
1451                                            % (p, revision)
1452                                            for p in self.depotPaths])):
1453
1454             if info['code'] == 'error':
1455                 sys.stderr.write("p4 returned an error: %s\n"
1456                                  % info['data'])
1457                 sys.exit(1)
1458
1459
1460             change = int(info["change"])
1461             if change > newestRevision:
1462                 newestRevision = change
1463
1464             if info["action"] in ("delete", "purge"):
1465                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1466                 #fileCnt = fileCnt + 1
1467                 continue
1468
1469             for prop in ["depotFile", "rev", "action", "type" ]:
1470                 details["%s%s" % (prop, fileCnt)] = info[prop]
1471
1472             fileCnt = fileCnt + 1
1473
1474         details["change"] = newestRevision
1475         self.updateOptionDict(details)
1476         try:
1477             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1478         except IOError:
1479             print "IO error with git fast-import. Is your git version recent enough?"
1480             print self.gitError.read()
1481
1482
1483     def getClientSpec(self):
1484         specList = p4CmdList( "client -o" )
1485         temp = {}
1486         for entry in specList:
1487             for k,v in entry.iteritems():
1488                 if k.startswith("View"):
1489                     if v.startswith('"'):
1490                         start = 1
1491                     else:
1492                         start = 0
1493                     index = v.find("...")
1494                     v = v[start:index]
1495                     if v.startswith("-"):
1496                         v = v[1:]
1497                         temp[v] = -len(v)
1498                     else:
1499                         temp[v] = len(v)
1500         self.clientSpecDirs = temp.items()
1501         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1502
1503     def run(self, args):
1504         self.depotPaths = []
1505         self.changeRange = ""
1506         self.initialParent = ""
1507         self.previousDepotPaths = []
1508
1509         # map from branch depot path to parent branch
1510         self.knownBranches = {}
1511         self.initialParents = {}
1512         self.hasOrigin = originP4BranchesExist()
1513         if not self.syncWithOrigin:
1514             self.hasOrigin = False
1515
1516         if self.importIntoRemotes:
1517             self.refPrefix = "refs/remotes/p4/"
1518         else:
1519             self.refPrefix = "refs/heads/p4/"
1520
1521         if self.syncWithOrigin and self.hasOrigin:
1522             if not self.silent:
1523                 print "Syncing with origin first by calling git fetch origin"
1524             system("git fetch origin")
1525
1526         if len(self.branch) == 0:
1527             self.branch = self.refPrefix + "master"
1528             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1529                 system("git update-ref %s refs/heads/p4" % self.branch)
1530                 system("git branch -D p4");
1531             # create it /after/ importing, when master exists
1532             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1533                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1534
1535         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1536             self.getClientSpec()
1537
1538         # TODO: should always look at previous commits,
1539         # merge with previous imports, if possible.
1540         if args == []:
1541             if self.hasOrigin:
1542                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1543             self.listExistingP4GitBranches()
1544
1545             if len(self.p4BranchesInGit) > 1:
1546                 if not self.silent:
1547                     print "Importing from/into multiple branches"
1548                 self.detectBranches = True
1549
1550             if self.verbose:
1551                 print "branches: %s" % self.p4BranchesInGit
1552
1553             p4Change = 0
1554             for branch in self.p4BranchesInGit:
1555                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1556
1557                 settings = extractSettingsGitLog(logMsg)
1558
1559                 self.readOptions(settings)
1560                 if (settings.has_key('depot-paths')
1561                     and settings.has_key ('change')):
1562                     change = int(settings['change']) + 1
1563                     p4Change = max(p4Change, change)
1564
1565                     depotPaths = sorted(settings['depot-paths'])
1566                     if self.previousDepotPaths == []:
1567                         self.previousDepotPaths = depotPaths
1568                     else:
1569                         paths = []
1570                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1571                             for i in range(0, min(len(cur), len(prev))):
1572                                 if cur[i] <> prev[i]:
1573                                     i = i - 1
1574                                     break
1575
1576                             paths.append (cur[:i + 1])
1577
1578                         self.previousDepotPaths = paths
1579
1580             if p4Change > 0:
1581                 self.depotPaths = sorted(self.previousDepotPaths)
1582                 self.changeRange = "@%s,#head" % p4Change
1583                 if not self.detectBranches:
1584                     self.initialParent = parseRevision(self.branch)
1585                 if not self.silent and not self.detectBranches:
1586                     print "Performing incremental import into %s git branch" % self.branch
1587
1588         if not self.branch.startswith("refs/"):
1589             self.branch = "refs/heads/" + self.branch
1590
1591         if len(args) == 0 and self.depotPaths:
1592             if not self.silent:
1593                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1594         else:
1595             if self.depotPaths and self.depotPaths != args:
1596                 print ("previous import used depot path %s and now %s was specified. "
1597                        "This doesn't work!" % (' '.join (self.depotPaths),
1598                                                ' '.join (args)))
1599                 sys.exit(1)
1600
1601             self.depotPaths = sorted(args)
1602
1603         revision = ""
1604         self.users = {}
1605
1606         newPaths = []
1607         for p in self.depotPaths:
1608             if p.find("@") != -1:
1609                 atIdx = p.index("@")
1610                 self.changeRange = p[atIdx:]
1611                 if self.changeRange == "@all":
1612                     self.changeRange = ""
1613                 elif ',' not in self.changeRange:
1614                     revision = self.changeRange
1615                     self.changeRange = ""
1616                 p = p[:atIdx]
1617             elif p.find("#") != -1:
1618                 hashIdx = p.index("#")
1619                 revision = p[hashIdx:]
1620                 p = p[:hashIdx]
1621             elif self.previousDepotPaths == []:
1622                 revision = "#head"
1623
1624             p = re.sub ("\.\.\.$", "", p)
1625             if not p.endswith("/"):
1626                 p += "/"
1627
1628             newPaths.append(p)
1629
1630         self.depotPaths = newPaths
1631
1632
1633         self.loadUserMapFromCache()
1634         self.labels = {}
1635         if self.detectLabels:
1636             self.getLabels();
1637
1638         if self.detectBranches:
1639             ## FIXME - what's a P4 projectName ?
1640             self.projectName = self.guessProjectName()
1641
1642             if self.hasOrigin:
1643                 self.getBranchMappingFromGitBranches()
1644             else:
1645                 self.getBranchMapping()
1646             if self.verbose:
1647                 print "p4-git branches: %s" % self.p4BranchesInGit
1648                 print "initial parents: %s" % self.initialParents
1649             for b in self.p4BranchesInGit:
1650                 if b != "master":
1651
1652                     ## FIXME
1653                     b = b[len(self.projectName):]
1654                 self.createdBranches.add(b)
1655
1656         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1657
1658         importProcess = subprocess.Popen(["git", "fast-import"],
1659                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1660                                          stderr=subprocess.PIPE);
1661         self.gitOutput = importProcess.stdout
1662         self.gitStream = importProcess.stdin
1663         self.gitError = importProcess.stderr
1664
1665         if revision:
1666             self.importHeadRevision(revision)
1667         else:
1668             changes = []
1669
1670             if len(self.changesFile) > 0:
1671                 output = open(self.changesFile).readlines()
1672                 changeSet = set()
1673                 for line in output:
1674                     changeSet.add(int(line))
1675
1676                 for change in changeSet:
1677                     changes.append(change)
1678
1679                 changes.sort()
1680             else:
1681                 if self.verbose:
1682                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1683                                                               self.changeRange)
1684                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1685
1686                 if len(self.maxChanges) > 0:
1687                     changes = changes[:min(int(self.maxChanges), len(changes))]
1688
1689             if len(changes) == 0:
1690                 if not self.silent:
1691                     print "No changes to import!"
1692                 return True
1693
1694             if not self.silent and not self.detectBranches:
1695                 print "Import destination: %s" % self.branch
1696
1697             self.updatedBranches = set()
1698
1699             self.importChanges(changes)
1700
1701             if not self.silent:
1702                 print ""
1703                 if len(self.updatedBranches) > 0:
1704                     sys.stdout.write("Updated branches: ")
1705                     for b in self.updatedBranches:
1706                         sys.stdout.write("%s " % b)
1707                     sys.stdout.write("\n")
1708
1709         self.gitStream.close()
1710         if importProcess.wait() != 0:
1711             die("fast-import failed: %s" % self.gitError.read())
1712         self.gitOutput.close()
1713         self.gitError.close()
1714
1715         return True
1716
1717 class P4Rebase(Command):
1718     def __init__(self):
1719         Command.__init__(self)
1720         self.options = [ ]
1721         self.description = ("Fetches the latest revision from perforce and "
1722                             + "rebases the current work (branch) against it")
1723         self.verbose = False
1724
1725     def run(self, args):
1726         sync = P4Sync()
1727         sync.run([])
1728
1729         return self.rebase()
1730
1731     def rebase(self):
1732         if os.system("git update-index --refresh") != 0:
1733             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1734         if len(read_pipe("git diff-index HEAD --")) > 0:
1735             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1736
1737         [upstream, settings] = findUpstreamBranchPoint()
1738         if len(upstream) == 0:
1739             die("Cannot find upstream branchpoint for rebase")
1740
1741         # the branchpoint may be p4/foo~3, so strip off the parent
1742         upstream = re.sub("~[0-9]+$", "", upstream)
1743
1744         print "Rebasing the current branch onto %s" % upstream
1745         oldHead = read_pipe("git rev-parse HEAD").strip()
1746         system("git rebase %s" % upstream)
1747         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1748         return True
1749
1750 class P4Clone(P4Sync):
1751     def __init__(self):
1752         P4Sync.__init__(self)
1753         self.description = "Creates a new git repository and imports from Perforce into it"
1754         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1755         self.options += [
1756             optparse.make_option("--destination", dest="cloneDestination",
1757                                  action='store', default=None,
1758                                  help="where to leave result of the clone"),
1759             optparse.make_option("-/", dest="cloneExclude",
1760                                  action="append", type="string",
1761                                  help="exclude depot path")
1762         ]
1763         self.cloneDestination = None
1764         self.needsGit = False
1765
1766     # This is required for the "append" cloneExclude action
1767     def ensure_value(self, attr, value):
1768         if not hasattr(self, attr) or getattr(self, attr) is None:
1769             setattr(self, attr, value)
1770         return getattr(self, attr)
1771
1772     def defaultDestination(self, args):
1773         ## TODO: use common prefix of args?
1774         depotPath = args[0]
1775         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1776         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1777         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1778         depotDir = re.sub(r"/$", "", depotDir)
1779         return os.path.split(depotDir)[1]
1780
1781     def run(self, args):
1782         if len(args) < 1:
1783             return False
1784
1785         if self.keepRepoPath and not self.cloneDestination:
1786             sys.stderr.write("Must specify destination for --keep-path\n")
1787             sys.exit(1)
1788
1789         depotPaths = args
1790
1791         if not self.cloneDestination and len(depotPaths) > 1:
1792             self.cloneDestination = depotPaths[-1]
1793             depotPaths = depotPaths[:-1]
1794
1795         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1796         for p in depotPaths:
1797             if not p.startswith("//"):
1798                 return False
1799
1800         if not self.cloneDestination:
1801             self.cloneDestination = self.defaultDestination(args)
1802
1803         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1804         if not os.path.exists(self.cloneDestination):
1805             os.makedirs(self.cloneDestination)
1806         chdir(self.cloneDestination)
1807         system("git init")
1808         self.gitdir = os.getcwd() + "/.git"
1809         if not P4Sync.run(self, depotPaths):
1810             return False
1811         if self.branch != "master":
1812             if self.importIntoRemotes:
1813                 masterbranch = "refs/remotes/p4/master"
1814             else:
1815                 masterbranch = "refs/heads/p4/master"
1816             if gitBranchExists(masterbranch):
1817                 system("git branch master %s" % masterbranch)
1818                 system("git checkout -f")
1819             else:
1820                 print "Could not detect main branch. No checkout/master branch created."
1821
1822         return True
1823
1824 class P4Branches(Command):
1825     def __init__(self):
1826         Command.__init__(self)
1827         self.options = [ ]
1828         self.description = ("Shows the git branches that hold imports and their "
1829                             + "corresponding perforce depot paths")
1830         self.verbose = False
1831
1832     def run(self, args):
1833         if originP4BranchesExist():
1834             createOrUpdateBranchesFromOrigin()
1835
1836         cmdline = "git rev-parse --symbolic "
1837         cmdline += " --remotes"
1838
1839         for line in read_pipe_lines(cmdline):
1840             line = line.strip()
1841
1842             if not line.startswith('p4/') or line == "p4/HEAD":
1843                 continue
1844             branch = line
1845
1846             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1847             settings = extractSettingsGitLog(log)
1848
1849             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1850         return True
1851
1852 class HelpFormatter(optparse.IndentedHelpFormatter):
1853     def __init__(self):
1854         optparse.IndentedHelpFormatter.__init__(self)
1855
1856     def format_description(self, description):
1857         if description:
1858             return description + "\n"
1859         else:
1860             return ""
1861
1862 def printUsage(commands):
1863     print "usage: %s <command> [options]" % sys.argv[0]
1864     print ""
1865     print "valid commands: %s" % ", ".join(commands)
1866     print ""
1867     print "Try %s <command> --help for command specific help." % sys.argv[0]
1868     print ""
1869
1870 commands = {
1871     "debug" : P4Debug,
1872     "submit" : P4Submit,
1873     "commit" : P4Submit,
1874     "sync" : P4Sync,
1875     "rebase" : P4Rebase,
1876     "clone" : P4Clone,
1877     "rollback" : P4RollBack,
1878     "branches" : P4Branches
1879 }
1880
1881
1882 def main():
1883     if len(sys.argv[1:]) == 0:
1884         printUsage(commands.keys())
1885         sys.exit(2)
1886
1887     cmd = ""
1888     cmdName = sys.argv[1]
1889     try:
1890         klass = commands[cmdName]
1891         cmd = klass()
1892     except KeyError:
1893         print "unknown command %s" % cmdName
1894         print ""
1895         printUsage(commands.keys())
1896         sys.exit(2)
1897
1898     options = cmd.options
1899     cmd.gitdir = os.environ.get("GIT_DIR", None)
1900
1901     args = sys.argv[2:]
1902
1903     if len(options) > 0:
1904         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1905
1906         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1907                                        options,
1908                                        description = cmd.description,
1909                                        formatter = HelpFormatter())
1910
1911         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1912     global verbose
1913     verbose = cmd.verbose
1914     if cmd.needsGit:
1915         if cmd.gitdir == None:
1916             cmd.gitdir = os.path.abspath(".git")
1917             if not isValidGitDir(cmd.gitdir):
1918                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1919                 if os.path.exists(cmd.gitdir):
1920                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1921                     if len(cdup) > 0:
1922                         chdir(cdup);
1923
1924         if not isValidGitDir(cmd.gitdir):
1925             if isValidGitDir(cmd.gitdir + "/.git"):
1926                 cmd.gitdir += "/.git"
1927             else:
1928                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1929
1930         os.environ["GIT_DIR"] = cmd.gitdir
1931
1932     if not cmd.run(args):
1933         parser.print_help()
1934
1935
1936 if __name__ == '__main__':
1937     main()