contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key):
 337     if not _gitConfig.has_key(key):
 338         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339     return _gitConfig[key]
 340
 341 def p4BranchesInGit(branchesAreInRemotes = True):
 342     branches = {}
 343
 344     cmdline = "git rev-parse --symbolic "
 345     if branchesAreInRemotes:
 346         cmdline += " --remotes"
 347     else:
 348         cmdline += " --branches"
 349
 350     for line in read_pipe_lines(cmdline):
 351         line = line.strip()
 352
 353         ## only import to p4/
 354         if not line.startswith('p4/') or line == "p4/HEAD":
 355             continue
 356         branch = line
 357
 358         # strip off p4
 359         branch = re.sub ("^p4/", "", line)
 360
 361         branches[branch] = parseRevision(line)
 362     return branches
 363
 364 def findUpstreamBranchPoint(head = "HEAD"):
 365     branches = p4BranchesInGit()
 366     # map from depot-path to branch name
 367     branchByDepotPath = {}
 368     for branch in branches.keys():
 369         tip = branches[branch]
 370         log = extractLogMessageFromGitCommit(tip)
 371         settings = extractSettingsGitLog(log)
 372         if settings.has_key("depot-paths"):
 373             paths = ",".join(settings["depot-paths"])
 374             branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376     settings = None
 377     parent = 0
 378     while parent < 65535:
 379         commit = head + "~%s" % parent
 380         log = extractLogMessageFromGitCommit(commit)
 381         settings = extractSettingsGitLog(log)
 382         if settings.has_key("depot-paths"):
 383             paths = ",".join(settings["depot-paths"])
 384             if branchByDepotPath.has_key(paths):
 385                 return [branchByDepotPath[paths], settings]
 386
 387         parent = parent + 1
 388
 389     return ["", settings]
 390
 391 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392     if not silent:
 393         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394                % localRefPrefix)
 395
 396     originPrefix = "origin/p4/"
 397
 398     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399         line = line.strip()
 400         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401             continue
 402
 403         headName = line[len(originPrefix):]
 404         remoteHead = localRefPrefix + headName
 405         originHead = line
 406
 407         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408         if (not original.has_key('depot-paths')
 409             or not original.has_key('change')):
 410             continue
 411
 412         update = False
 413         if not gitBranchExists(remoteHead):
 414             if verbose:
 415                 print "creating %s" % remoteHead
 416             update = True
 417         else:
 418             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419             if settings.has_key('change') > 0:
 420                 if settings['depot-paths'] == original['depot-paths']:
 421                     originP4Change = int(original['change'])
 422                     p4Change = int(settings['change'])
 423                     if originP4Change > p4Change:
 424                         print ("%s (%s) is newer than %s (%s). "
 425                                "Updating p4 branch from origin."
 426                                % (originHead, originP4Change,
 427                                   remoteHead, p4Change))
 428                         update = True
 429                 else:
 430                     print ("Ignoring: %s was imported from %s while "
 431                            "%s was imported from %s"
 432                            % (originHead, ','.join(original['depot-paths']),
 433                               remoteHead, ','.join(settings['depot-paths'])))
 434
 435         if update:
 436             system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438 def originP4BranchesExist():
 439         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441 def p4ChangesForPaths(depotPaths, changeRange):
 442     assert depotPaths
 443     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                         for p in depotPaths]))
 445
 446     changes = {}
 447     for line in output:
 448         changeNum = int(line.split(" ")[1])
 449         changes[changeNum] = True
 450
 451     changelist = changes.keys()
 452     changelist.sort()
 453     return changelist
 454
 455 class Command:
 456     def __init__(self):
 457         self.usage = "usage: %prog [options]"
 458         self.needsGit = True
 459
 460 class P4Debug(Command):
 461     def __init__(self):
 462         Command.__init__(self)
 463         self.options = [
 464             optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                  default=False),
 466             ]
 467         self.description = "A tool to debug the output of p4 -G."
 468         self.needsGit = False
 469         self.verbose = False
 470
 471     def run(self, args):
 472         j = 0
 473         for output in p4CmdList(" ".join(args)):
 474             print 'Element: %d' % j
 475             j += 1
 476             print output
 477         return True
 478
 479 class P4RollBack(Command):
 480     def __init__(self):
 481         Command.__init__(self)
 482         self.options = [
 483             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485         ]
 486         self.description = "A tool to debug the multi-branch import. Don't use :)"
 487         self.verbose = False
 488         self.rollbackLocalBranches = False
 489
 490     def run(self, args):
 491         if len(args) != 1:
 492             return False
 493         maxChange = int(args[0])
 494
 495         if "p4ExitCode" in p4Cmd("changes -m 1"):
 496             die("Problems executing p4");
 497
 498         if self.rollbackLocalBranches:
 499             refPrefix = "refs/heads/"
 500             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501         else:
 502             refPrefix = "refs/remotes/"
 503             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505         for line in lines:
 506             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                 line = line.strip()
 508                 ref = refPrefix + line
 509                 log = extractLogMessageFromGitCommit(ref)
 510                 settings = extractSettingsGitLog(log)
 511
 512                 depotPaths = settings['depot-paths']
 513                 change = settings['change']
 514
 515                 changed = False
 516
 517                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                            for p in depotPaths]))) == 0:
 519                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                     continue
 522
 523                 while change and int(change) > maxChange:
 524                     changed = True
 525                     if self.verbose:
 526                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                     system("git update-ref %s \"%s^\"" % (ref, ref))
 528                     log = extractLogMessageFromGitCommit(ref)
 529                     settings =  extractSettingsGitLog(log)
 530
 531
 532                     depotPaths = settings['depot-paths']
 533                     change = settings['change']
 534
 535                 if changed:
 536                     print "%s rewound to %s" % (ref, change)
 537
 538         return True
 539
 540 class P4Submit(Command):
 541     def __init__(self):
 542         Command.__init__(self)
 543         self.options = [
 544                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                 optparse.make_option("--origin", dest="origin"),
 546                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 547         ]
 548         self.description = "Submit changes from git to the perforce depot."
 549         self.usage += " [name of git branch to submit into perforce depot]"
 550         self.interactive = True
 551         self.origin = ""
 552         self.detectRename = False
 553         self.verbose = False
 554         self.isWindows = (platform.system() == "Windows")
 555
 556     def check(self):
 557         if len(p4CmdList("opened ...")) > 0:
 558             die("You have files opened with perforce! Close them before starting the sync.")
 559
 560     # replaces everything between 'Description:' and the next P4 submit template field with the
 561     # commit message
 562     def prepareLogMessage(self, template, message):
 563         result = ""
 564
 565         inDescriptionSection = False
 566
 567         for line in template.split("\n"):
 568             if line.startswith("#"):
 569                 result += line + "\n"
 570                 continue
 571
 572             if inDescriptionSection:
 573                 if line.startswith("Files:"):
 574                     inDescriptionSection = False
 575                 else:
 576                     continue
 577             else:
 578                 if line.startswith("Description:"):
 579                     inDescriptionSection = True
 580                     line += "\n"
 581                     for messageLine in message.split("\n"):
 582                         line += "\t" + messageLine + "\n"
 583
 584             result += line + "\n"
 585
 586         return result
 587
 588     def prepareSubmitTemplate(self):
 589         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590         template = ""
 591         inFilesSection = False
 592         for line in p4_read_pipe_lines("change -o"):
 593             if line.endswith("\r\n"):
 594                 line = line[:-2] + "\n"
 595             if inFilesSection:
 596                 if line.startswith("\t"):
 597                     # path starts and ends with a tab
 598                     path = line[1:]
 599                     lastTab = path.rfind("\t")
 600                     if lastTab != -1:
 601                         path = path[:lastTab]
 602                         if not path.startswith(self.depotPath):
 603                             continue
 604                 else:
 605                     inFilesSection = False
 606             else:
 607                 if line.startswith("Files:"):
 608                     inFilesSection = True
 609
 610             template += line
 611
 612         return template
 613
 614     def applyCommit(self, id):
 615         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616         diffOpts = ("", "-M")[self.detectRename]
 617         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618         filesToAdd = set()
 619         filesToDelete = set()
 620         editedFiles = set()
 621         filesToChangeExecBit = {}
 622         for line in diff:
 623             diff = parseDiffTreeEntry(line)
 624             modifier = diff['status']
 625             path = diff['src']
 626             if modifier == "M":
 627                 p4_system("edit \"%s\"" % path)
 628                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                     filesToChangeExecBit[path] = diff['dst_mode']
 630                 editedFiles.add(path)
 631             elif modifier == "A":
 632                 filesToAdd.add(path)
 633                 filesToChangeExecBit[path] = diff['dst_mode']
 634                 if path in filesToDelete:
 635                     filesToDelete.remove(path)
 636             elif modifier == "D":
 637                 filesToDelete.add(path)
 638                 if path in filesToAdd:
 639                     filesToAdd.remove(path)
 640             elif modifier == "R":
 641                 src, dest = diff['src'], diff['dst']
 642                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                 p4_system("edit \"%s\"" % (dest))
 644                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                     filesToChangeExecBit[dest] = diff['dst_mode']
 646                 os.unlink(dest)
 647                 editedFiles.add(dest)
 648                 filesToDelete.add(src)
 649             else:
 650                 die("unknown modifier %s for %s" % (modifier, path))
 651
 652         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653         patchcmd = diffcmd + " | git apply "
 654         tryPatchCmd = patchcmd + "--check -"
 655         applyPatchCmd = patchcmd + "--check --apply -"
 656
 657         if os.system(tryPatchCmd) != 0:
 658             print "Unfortunately applying the change failed!"
 659             print "What do you want to do?"
 660             response = "x"
 661             while response != "s" and response != "a" and response != "w":
 662                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664             if response == "s":
 665                 print "Skipping! Good luck with the next patches..."
 666                 for f in editedFiles:
 667                     p4_system("revert \"%s\"" % f);
 668                 for f in filesToAdd:
 669                     system("rm %s" %f)
 670                 return
 671             elif response == "a":
 672                 os.system(applyPatchCmd)
 673                 if len(filesToAdd) > 0:
 674                     print "You may also want to call p4 add on the following files:"
 675                     print " ".join(filesToAdd)
 676                 if len(filesToDelete):
 677                     print "The following files should be scheduled for deletion with p4 delete:"
 678                     print " ".join(filesToDelete)
 679                 die("Please resolve and submit the conflict manually and "
 680                     + "continue afterwards with git-p4 submit --continue")
 681             elif response == "w":
 682                 system(diffcmd + " > patch.txt")
 683                 print "Patch saved to patch.txt in %s !" % self.clientPath
 684                 die("Please resolve and submit the conflict manually and "
 685                     "continue afterwards with git-p4 submit --continue")
 686
 687         system(applyPatchCmd)
 688
 689         for f in filesToAdd:
 690             p4_system("add \"%s\"" % f)
 691         for f in filesToDelete:
 692             p4_system("revert \"%s\"" % f)
 693             p4_system("delete \"%s\"" % f)
 694
 695         # Set/clear executable bits
 696         for f in filesToChangeExecBit.keys():
 697             mode = filesToChangeExecBit[f]
 698             setP4ExecBit(f, mode)
 699
 700         logMessage = extractLogMessageFromGitCommit(id)
 701         logMessage = logMessage.strip()
 702
 703         template = self.prepareSubmitTemplate()
 704
 705         if self.interactive:
 706             submitTemplate = self.prepareLogMessage(template, logMessage)
 707             if os.environ.has_key("P4DIFF"):
 708                 del(os.environ["P4DIFF"])
 709             diff = ""
 710             for editedFile in editedFiles:
 711                 diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713             newdiff = ""
 714             for newFile in filesToAdd:
 715                 newdiff += "==== new file ====\n"
 716                 newdiff += "--- /dev/null\n"
 717                 newdiff += "+++ %s\n" % newFile
 718                 f = open(newFile, "r")
 719                 for line in f.readlines():
 720                     newdiff += "+" + line
 721                 f.close()
 722
 723             separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725             [handle, fileName] = tempfile.mkstemp()
 726             tmpFile = os.fdopen(handle, "w+")
 727             if self.isWindows:
 728                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                 separatorLine = separatorLine.replace("\n", "\r\n")
 730                 newdiff = newdiff.replace("\n", "\r\n")
 731             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732             tmpFile.close()
 733             mtime = os.stat(fileName).st_mtime
 734             if os.environ.has_key("P4EDITOR"):
 735                 editor = os.environ.get("P4EDITOR")
 736             else:
 737                 editor = read_pipe("git var GIT_EDITOR").strip()
 738             system(editor + " " + fileName)
 739
 740             response = "y"
 741             if os.stat(fileName).st_mtime <= mtime:
 742                 response = "x"
 743                 while response != "y" and response != "n":
 744                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746             if response == "y":
 747                 tmpFile = open(fileName, "rb")
 748                 message = tmpFile.read()
 749                 tmpFile.close()
 750                 submitTemplate = message[:message.index(separatorLine)]
 751                 if self.isWindows:
 752                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                 p4_write_pipe("submit -i", submitTemplate)
 754             else:
 755                 for f in editedFiles:
 756                     p4_system("revert \"%s\"" % f);
 757                 for f in filesToAdd:
 758                     p4_system("revert \"%s\"" % f);
 759                     system("rm %s" %f)
 760
 761             os.remove(fileName)
 762         else:
 763             fileName = "submit.txt"
 764             file = open(fileName, "w+")
 765             file.write(self.prepareLogMessage(template, logMessage))
 766             file.close()
 767             print ("Perforce submit template written as %s. "
 768                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                    % (fileName, fileName))
 770
 771     def run(self, args):
 772         if len(args) == 0:
 773             self.master = currentGitBranch()
 774             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                 die("Detecting current git branch failed!")
 776         elif len(args) == 1:
 777             self.master = args[0]
 778         else:
 779             return False
 780
 781         allowSubmit = gitConfig("git-p4.allowSubmit")
 782         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783             die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785         [upstream, settings] = findUpstreamBranchPoint()
 786         self.depotPath = settings['depot-paths'][0]
 787         if len(self.origin) == 0:
 788             self.origin = upstream
 789
 790         if self.verbose:
 791             print "Origin branch is " + self.origin
 792
 793         if len(self.depotPath) == 0:
 794             print "Internal error: cannot locate perforce depot path from existing branches"
 795             sys.exit(128)
 796
 797         self.clientPath = p4Where(self.depotPath)
 798
 799         if len(self.clientPath) == 0:
 800             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801             sys.exit(128)
 802
 803         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804         self.oldWorkingDirectory = os.getcwd()
 805
 806         chdir(self.clientPath)
 807         print "Synchronizing p4 checkout..."
 808         p4_system("sync ...")
 809
 810         self.check()
 811
 812         commits = []
 813         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814             commits.append(line.strip())
 815         commits.reverse()
 816
 817         while len(commits) > 0:
 818             commit = commits[0]
 819             commits = commits[1:]
 820             self.applyCommit(commit)
 821             if not self.interactive:
 822                 break
 823
 824         if len(commits) == 0:
 825             print "All changes applied!"
 826             chdir(self.oldWorkingDirectory)
 827
 828             sync = P4Sync()
 829             sync.run([])
 830
 831             rebase = P4Rebase()
 832             rebase.rebase()
 833
 834         return True
 835
 836 class P4Sync(Command):
 837     def __init__(self):
 838         Command.__init__(self)
 839         self.options = [
 840                 optparse.make_option("--branch", dest="branch"),
 841                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 842                 optparse.make_option("--changesfile", dest="changesFile"),
 843                 optparse.make_option("--silent", dest="silent", action="store_true"),
 844                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 845                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 846                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 847                                      help="Import into refs/heads/ , not refs/remotes"),
 848                 optparse.make_option("--max-changes", dest="maxChanges"),
 849                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 850                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 851                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 852                                      help="Only sync files that are included in the Perforce Client Spec")
 853         ]
 854         self.description = """Imports from Perforce into a git repository.\n
 855     example:
 856     //depot/my/project/ -- to import the current head
 857     //depot/my/project/@all -- to import everything
 858     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 859
 860     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 861
 862         self.usage += " //depot/path[@revRange]"
 863         self.silent = False
 864         self.createdBranches = set()
 865         self.committedChanges = set()
 866         self.branch = ""
 867         self.detectBranches = False
 868         self.detectLabels = False
 869         self.changesFile = ""
 870         self.syncWithOrigin = True
 871         self.verbose = False
 872         self.importIntoRemotes = True
 873         self.maxChanges = ""
 874         self.isWindows = (platform.system() == "Windows")
 875         self.keepRepoPath = False
 876         self.depotPaths = None
 877         self.p4BranchesInGit = []
 878         self.cloneExclude = []
 879         self.useClientSpec = False
 880         self.clientSpecDirs = []
 881
 882         if gitConfig("git-p4.syncFromOrigin") == "false":
 883             self.syncWithOrigin = False
 884
 885     def extractFilesFromCommit(self, commit):
 886         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 887                              for path in self.cloneExclude]
 888         files = []
 889         fnum = 0
 890         while commit.has_key("depotFile%s" % fnum):
 891             path =  commit["depotFile%s" % fnum]
 892
 893             if [p for p in self.cloneExclude
 894                 if path.startswith (p)]:
 895                 found = False
 896             else:
 897                 found = [p for p in self.depotPaths
 898                          if path.startswith (p)]
 899             if not found:
 900                 fnum = fnum + 1
 901                 continue
 902
 903             file = {}
 904             file["path"] = path
 905             file["rev"] = commit["rev%s" % fnum]
 906             file["action"] = commit["action%s" % fnum]
 907             file["type"] = commit["type%s" % fnum]
 908             files.append(file)
 909             fnum = fnum + 1
 910         return files
 911
 912     def stripRepoPath(self, path, prefixes):
 913         if self.keepRepoPath:
 914             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 915
 916         for p in prefixes:
 917             if path.startswith(p):
 918                 path = path[len(p):]
 919
 920         return path
 921
 922     def splitFilesIntoBranches(self, commit):
 923         branches = {}
 924         fnum = 0
 925         while commit.has_key("depotFile%s" % fnum):
 926             path =  commit["depotFile%s" % fnum]
 927             found = [p for p in self.depotPaths
 928                      if path.startswith (p)]
 929             if not found:
 930                 fnum = fnum + 1
 931                 continue
 932
 933             file = {}
 934             file["path"] = path
 935             file["rev"] = commit["rev%s" % fnum]
 936             file["action"] = commit["action%s" % fnum]
 937             file["type"] = commit["type%s" % fnum]
 938             fnum = fnum + 1
 939
 940             relPath = self.stripRepoPath(path, self.depotPaths)
 941
 942             for branch in self.knownBranches.keys():
 943
 944                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 945                 if relPath.startswith(branch + "/"):
 946                     if branch not in branches:
 947                         branches[branch] = []
 948                     branches[branch].append(file)
 949                     break
 950
 951         return branches
 952
 953     # output one file from the P4 stream
 954     # - helper for streamP4Files
 955
 956     def streamOneP4File(self, file, contents):
 957         if file["type"] == "apple":
 958             print "\nfile %s is a strange apple file that forks. Ignoring" % \
 959                 file['depotFile']
 960             return
 961
 962         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 963         if verbose:
 964             sys.stderr.write("%s\n" % relPath)
 965
 966         mode = "644"
 967         if isP4Exec(file["type"]):
 968             mode = "755"
 969         elif file["type"] == "symlink":
 970             mode = "120000"
 971             # p4 print on a symlink contains "target\n", so strip it off
 972             data = ''.join(contents)
 973             contents = [data[:-1]]
 974
 975         if self.isWindows and file["type"].endswith("text"):
 976             mangled = []
 977             for data in contents:
 978                 data = data.replace("\r\n", "\n")
 979                 mangled.append(data)
 980             contents = mangled
 981
 982         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 983             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 984         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 985             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 986
 987         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 988
 989         # total length...
 990         length = 0
 991         for d in contents:
 992             length = length + len(d)
 993
 994         self.gitStream.write("data %d\n" % length)
 995         for d in contents:
 996             self.gitStream.write(d)
 997         self.gitStream.write("\n")
 998
 999     def streamOneP4Deletion(self, file):
1000         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1001         if verbose:
1002             sys.stderr.write("delete %s\n" % relPath)
1003         self.gitStream.write("D %s\n" % relPath)
1004
1005     # handle another chunk of streaming data
1006     def streamP4FilesCb(self, marshalled):
1007
1008         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1009             # start of a new file - output the old one first
1010             self.streamOneP4File(self.stream_file, self.stream_contents)
1011             self.stream_file = {}
1012             self.stream_contents = []
1013             self.stream_have_file_info = False
1014
1015         # pick up the new file information... for the
1016         # 'data' field we need to append to our array
1017         for k in marshalled.keys():
1018             if k == 'data':
1019                 self.stream_contents.append(marshalled['data'])
1020             else:
1021                 self.stream_file[k] = marshalled[k]
1022
1023         self.stream_have_file_info = True
1024
1025     # Stream directly from "p4 files" into "git fast-import"
1026     def streamP4Files(self, files):
1027         filesForCommit = []
1028         filesToRead = []
1029         filesToDelete = []
1030
1031         for f in files:
1032             includeFile = True
1033             for val in self.clientSpecDirs:
1034                 if f['path'].startswith(val[0]):
1035                     if val[1] <= 0:
1036                         includeFile = False
1037                     break
1038
1039             if includeFile:
1040                 filesForCommit.append(f)
1041                 if f['action'] not in ('delete', 'move/delete', 'purge'):
1042                     filesToRead.append(f)
1043                 else:
1044                     filesToDelete.append(f)
1045
1046         # deleted files...
1047         for f in filesToDelete:
1048             self.streamOneP4Deletion(f)
1049
1050         if len(filesToRead) > 0:
1051             self.stream_file = {}
1052             self.stream_contents = []
1053             self.stream_have_file_info = False
1054
1055             # curry self argument
1056             def streamP4FilesCbSelf(entry):
1057                 self.streamP4FilesCb(entry)
1058
1059             p4CmdList("-x - print",
1060                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1061                                                   for f in filesToRead]),
1062                 cb=streamP4FilesCbSelf)
1063
1064             # do the last chunk
1065             if self.stream_file.has_key('depotFile'):
1066                 self.streamOneP4File(self.stream_file, self.stream_contents)
1067
1068     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1069         epoch = details["time"]
1070         author = details["user"]
1071         self.branchPrefixes = branchPrefixes
1072
1073         if self.verbose:
1074             print "commit into %s" % branch
1075
1076         # start with reading files; if that fails, we should not
1077         # create a commit.
1078         new_files = []
1079         for f in files:
1080             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1081                 new_files.append (f)
1082             else:
1083                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1084
1085         self.gitStream.write("commit %s\n" % branch)
1086 #        gitStream.write("mark :%s\n" % details["change"])
1087         self.committedChanges.add(int(details["change"]))
1088         committer = ""
1089         if author not in self.users:
1090             self.getUserMapFromPerforceServer()
1091         if author in self.users:
1092             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1093         else:
1094             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1095
1096         self.gitStream.write("committer %s\n" % committer)
1097
1098         self.gitStream.write("data <<EOT\n")
1099         self.gitStream.write(details["desc"])
1100         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1101                              % (','.join (branchPrefixes), details["change"]))
1102         if len(details['options']) > 0:
1103             self.gitStream.write(": options = %s" % details['options'])
1104         self.gitStream.write("]\nEOT\n\n")
1105
1106         if len(parent) > 0:
1107             if self.verbose:
1108                 print "parent %s" % parent
1109             self.gitStream.write("from %s\n" % parent)
1110
1111         self.streamP4Files(new_files)
1112         self.gitStream.write("\n")
1113
1114         change = int(details["change"])
1115
1116         if self.labels.has_key(change):
1117             label = self.labels[change]
1118             labelDetails = label[0]
1119             labelRevisions = label[1]
1120             if self.verbose:
1121                 print "Change %s is labelled %s" % (change, labelDetails)
1122
1123             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1124                                                     for p in branchPrefixes]))
1125
1126             if len(files) == len(labelRevisions):
1127
1128                 cleanedFiles = {}
1129                 for info in files:
1130                     if info["action"] in ("delete", "purge"):
1131                         continue
1132                     cleanedFiles[info["depotFile"]] = info["rev"]
1133
1134                 if cleanedFiles == labelRevisions:
1135                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1136                     self.gitStream.write("from %s\n" % branch)
1137
1138                     owner = labelDetails["Owner"]
1139                     tagger = ""
1140                     if author in self.users:
1141                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1142                     else:
1143                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1144                     self.gitStream.write("tagger %s\n" % tagger)
1145                     self.gitStream.write("data <<EOT\n")
1146                     self.gitStream.write(labelDetails["Description"])
1147                     self.gitStream.write("EOT\n\n")
1148
1149                 else:
1150                     if not self.silent:
1151                         print ("Tag %s does not match with change %s: files do not match."
1152                                % (labelDetails["label"], change))
1153
1154             else:
1155                 if not self.silent:
1156                     print ("Tag %s does not match with change %s: file count is different."
1157                            % (labelDetails["label"], change))
1158
1159     def getUserCacheFilename(self):
1160         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1161         return home + "/.gitp4-usercache.txt"
1162
1163     def getUserMapFromPerforceServer(self):
1164         if self.userMapFromPerforceServer:
1165             return
1166         self.users = {}
1167
1168         for output in p4CmdList("users"):
1169             if not output.has_key("User"):
1170                 continue
1171             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1172
1173
1174         s = ''
1175         for (key, val) in self.users.items():
1176             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1177
1178         open(self.getUserCacheFilename(), "wb").write(s)
1179         self.userMapFromPerforceServer = True
1180
1181     def loadUserMapFromCache(self):
1182         self.users = {}
1183         self.userMapFromPerforceServer = False
1184         try:
1185             cache = open(self.getUserCacheFilename(), "rb")
1186             lines = cache.readlines()
1187             cache.close()
1188             for line in lines:
1189                 entry = line.strip().split("\t")
1190                 self.users[entry[0]] = entry[1]
1191         except IOError:
1192             self.getUserMapFromPerforceServer()
1193
1194     def getLabels(self):
1195         self.labels = {}
1196
1197         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1198         if len(l) > 0 and not self.silent:
1199             print "Finding files belonging to labels in %s" % `self.depotPaths`
1200
1201         for output in l:
1202             label = output["label"]
1203             revisions = {}
1204             newestChange = 0
1205             if self.verbose:
1206                 print "Querying files for label %s" % label
1207             for file in p4CmdList("files "
1208                                   +  ' '.join (["%s...@%s" % (p, label)
1209                                                 for p in self.depotPaths])):
1210                 revisions[file["depotFile"]] = file["rev"]
1211                 change = int(file["change"])
1212                 if change > newestChange:
1213                     newestChange = change
1214
1215             self.labels[newestChange] = [output, revisions]
1216
1217         if self.verbose:
1218             print "Label changes: %s" % self.labels.keys()
1219
1220     def guessProjectName(self):
1221         for p in self.depotPaths:
1222             if p.endswith("/"):
1223                 p = p[:-1]
1224             p = p[p.strip().rfind("/") + 1:]
1225             if not p.endswith("/"):
1226                p += "/"
1227             return p
1228
1229     def getBranchMapping(self):
1230         lostAndFoundBranches = set()
1231
1232         for info in p4CmdList("branches"):
1233             details = p4Cmd("branch -o %s" % info["branch"])
1234             viewIdx = 0
1235             while details.has_key("View%s" % viewIdx):
1236                 paths = details["View%s" % viewIdx].split(" ")
1237                 viewIdx = viewIdx + 1
1238                 # require standard //depot/foo/... //depot/bar/... mapping
1239                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1240                     continue
1241                 source = paths[0]
1242                 destination = paths[1]
1243                 ## HACK
1244                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1245                     source = source[len(self.depotPaths[0]):-4]
1246                     destination = destination[len(self.depotPaths[0]):-4]
1247
1248                     if destination in self.knownBranches:
1249                         if not self.silent:
1250                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1251                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1252                         continue
1253
1254                     self.knownBranches[destination] = source
1255
1256                     lostAndFoundBranches.discard(destination)
1257
1258                     if source not in self.knownBranches:
1259                         lostAndFoundBranches.add(source)
1260
1261
1262         for branch in lostAndFoundBranches:
1263             self.knownBranches[branch] = branch
1264
1265     def getBranchMappingFromGitBranches(self):
1266         branches = p4BranchesInGit(self.importIntoRemotes)
1267         for branch in branches.keys():
1268             if branch == "master":
1269                 branch = "main"
1270             else:
1271                 branch = branch[len(self.projectName):]
1272             self.knownBranches[branch] = branch
1273
1274     def listExistingP4GitBranches(self):
1275         # branches holds mapping from name to commit
1276         branches = p4BranchesInGit(self.importIntoRemotes)
1277         self.p4BranchesInGit = branches.keys()
1278         for branch in branches.keys():
1279             self.initialParents[self.refPrefix + branch] = branches[branch]
1280
1281     def updateOptionDict(self, d):
1282         option_keys = {}
1283         if self.keepRepoPath:
1284             option_keys['keepRepoPath'] = 1
1285
1286         d["options"] = ' '.join(sorted(option_keys.keys()))
1287
1288     def readOptions(self, d):
1289         self.keepRepoPath = (d.has_key('options')
1290                              and ('keepRepoPath' in d['options']))
1291
1292     def gitRefForBranch(self, branch):
1293         if branch == "main":
1294             return self.refPrefix + "master"
1295
1296         if len(branch) <= 0:
1297             return branch
1298
1299         return self.refPrefix + self.projectName + branch
1300
1301     def gitCommitByP4Change(self, ref, change):
1302         if self.verbose:
1303             print "looking in ref " + ref + " for change %s using bisect..." % change
1304
1305         earliestCommit = ""
1306         latestCommit = parseRevision(ref)
1307
1308         while True:
1309             if self.verbose:
1310                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1311             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1312             if len(next) == 0:
1313                 if self.verbose:
1314                     print "argh"
1315                 return ""
1316             log = extractLogMessageFromGitCommit(next)
1317             settings = extractSettingsGitLog(log)
1318             currentChange = int(settings['change'])
1319             if self.verbose:
1320                 print "current change %s" % currentChange
1321
1322             if currentChange == change:
1323                 if self.verbose:
1324                     print "found %s" % next
1325                 return next
1326
1327             if currentChange < change:
1328                 earliestCommit = "^%s" % next
1329             else:
1330                 latestCommit = "%s" % next
1331
1332         return ""
1333
1334     def importNewBranch(self, branch, maxChange):
1335         # make fast-import flush all changes to disk and update the refs using the checkpoint
1336         # command so that we can try to find the branch parent in the git history
1337         self.gitStream.write("checkpoint\n\n");
1338         self.gitStream.flush();
1339         branchPrefix = self.depotPaths[0] + branch + "/"
1340         range = "@1,%s" % maxChange
1341         #print "prefix" + branchPrefix
1342         changes = p4ChangesForPaths([branchPrefix], range)
1343         if len(changes) <= 0:
1344             return False
1345         firstChange = changes[0]
1346         #print "first change in branch: %s" % firstChange
1347         sourceBranch = self.knownBranches[branch]
1348         sourceDepotPath = self.depotPaths[0] + sourceBranch
1349         sourceRef = self.gitRefForBranch(sourceBranch)
1350         #print "source " + sourceBranch
1351
1352         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1353         #print "branch parent: %s" % branchParentChange
1354         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1355         if len(gitParent) > 0:
1356             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1357             #print "parent git commit: %s" % gitParent
1358
1359         self.importChanges(changes)
1360         return True
1361
1362     def importChanges(self, changes):
1363         cnt = 1
1364         for change in changes:
1365             description = p4Cmd("describe %s" % change)
1366             self.updateOptionDict(description)
1367
1368             if not self.silent:
1369                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1370                 sys.stdout.flush()
1371             cnt = cnt + 1
1372
1373             try:
1374                 if self.detectBranches:
1375                     branches = self.splitFilesIntoBranches(description)
1376                     for branch in branches.keys():
1377                         ## HACK  --hwn
1378                         branchPrefix = self.depotPaths[0] + branch + "/"
1379
1380                         parent = ""
1381
1382                         filesForCommit = branches[branch]
1383
1384                         if self.verbose:
1385                             print "branch is %s" % branch
1386
1387                         self.updatedBranches.add(branch)
1388
1389                         if branch not in self.createdBranches:
1390                             self.createdBranches.add(branch)
1391                             parent = self.knownBranches[branch]
1392                             if parent == branch:
1393                                 parent = ""
1394                             else:
1395                                 fullBranch = self.projectName + branch
1396                                 if fullBranch not in self.p4BranchesInGit:
1397                                     if not self.silent:
1398                                         print("\n    Importing new branch %s" % fullBranch);
1399                                     if self.importNewBranch(branch, change - 1):
1400                                         parent = ""
1401                                         self.p4BranchesInGit.append(fullBranch)
1402                                     if not self.silent:
1403                                         print("\n    Resuming with change %s" % change);
1404
1405                                 if self.verbose:
1406                                     print "parent determined through known branches: %s" % parent
1407
1408                         branch = self.gitRefForBranch(branch)
1409                         parent = self.gitRefForBranch(parent)
1410
1411                         if self.verbose:
1412                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1413
1414                         if len(parent) == 0 and branch in self.initialParents:
1415                             parent = self.initialParents[branch]
1416                             del self.initialParents[branch]
1417
1418                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1419                 else:
1420                     files = self.extractFilesFromCommit(description)
1421                     self.commit(description, files, self.branch, self.depotPaths,
1422                                 self.initialParent)
1423                     self.initialParent = ""
1424             except IOError:
1425                 print self.gitError.read()
1426                 sys.exit(1)
1427
1428     def importHeadRevision(self, revision):
1429         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1430
1431         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1432         details["desc"] = ("Initial import of %s from the state at revision %s"
1433                            % (' '.join(self.depotPaths), revision))
1434         details["change"] = revision
1435         newestRevision = 0
1436
1437         fileCnt = 0
1438         for info in p4CmdList("files "
1439                               +  ' '.join(["%s...%s"
1440                                            % (p, revision)
1441                                            for p in self.depotPaths])):
1442
1443             if 'code' in info and info['code'] == 'error':
1444                 sys.stderr.write("p4 returned an error: %s\n"
1445                                  % info['data'])
1446                 sys.exit(1)
1447             if 'p4ExitCode' in info:
1448                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1449                 sys.exit(1)
1450
1451
1452             change = int(info["change"])
1453             if change > newestRevision:
1454                 newestRevision = change
1455
1456             if info["action"] in ("delete", "purge"):
1457                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1458                 #fileCnt = fileCnt + 1
1459                 continue
1460
1461             for prop in ["depotFile", "rev", "action", "type" ]:
1462                 details["%s%s" % (prop, fileCnt)] = info[prop]
1463
1464             fileCnt = fileCnt + 1
1465
1466         details["change"] = newestRevision
1467         self.updateOptionDict(details)
1468         try:
1469             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1470         except IOError:
1471             print "IO error with git fast-import. Is your git version recent enough?"
1472             print self.gitError.read()
1473
1474
1475     def getClientSpec(self):
1476         specList = p4CmdList( "client -o" )
1477         temp = {}
1478         for entry in specList:
1479             for k,v in entry.iteritems():
1480                 if k.startswith("View"):
1481                     if v.startswith('"'):
1482                         start = 1
1483                     else:
1484                         start = 0
1485                     index = v.find("...")
1486                     v = v[start:index]
1487                     if v.startswith("-"):
1488                         v = v[1:]
1489                         temp[v] = -len(v)
1490                     else:
1491                         temp[v] = len(v)
1492         self.clientSpecDirs = temp.items()
1493         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1494
1495     def run(self, args):
1496         self.depotPaths = []
1497         self.changeRange = ""
1498         self.initialParent = ""
1499         self.previousDepotPaths = []
1500
1501         # map from branch depot path to parent branch
1502         self.knownBranches = {}
1503         self.initialParents = {}
1504         self.hasOrigin = originP4BranchesExist()
1505         if not self.syncWithOrigin:
1506             self.hasOrigin = False
1507
1508         if self.importIntoRemotes:
1509             self.refPrefix = "refs/remotes/p4/"
1510         else:
1511             self.refPrefix = "refs/heads/p4/"
1512
1513         if self.syncWithOrigin and self.hasOrigin:
1514             if not self.silent:
1515                 print "Syncing with origin first by calling git fetch origin"
1516             system("git fetch origin")
1517
1518         if len(self.branch) == 0:
1519             self.branch = self.refPrefix + "master"
1520             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1521                 system("git update-ref %s refs/heads/p4" % self.branch)
1522                 system("git branch -D p4");
1523             # create it /after/ importing, when master exists
1524             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1525                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1526
1527         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1528             self.getClientSpec()
1529
1530         # TODO: should always look at previous commits,
1531         # merge with previous imports, if possible.
1532         if args == []:
1533             if self.hasOrigin:
1534                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1535             self.listExistingP4GitBranches()
1536
1537             if len(self.p4BranchesInGit) > 1:
1538                 if not self.silent:
1539                     print "Importing from/into multiple branches"
1540                 self.detectBranches = True
1541
1542             if self.verbose:
1543                 print "branches: %s" % self.p4BranchesInGit
1544
1545             p4Change = 0
1546             for branch in self.p4BranchesInGit:
1547                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1548
1549                 settings = extractSettingsGitLog(logMsg)
1550
1551                 self.readOptions(settings)
1552                 if (settings.has_key('depot-paths')
1553                     and settings.has_key ('change')):
1554                     change = int(settings['change']) + 1
1555                     p4Change = max(p4Change, change)
1556
1557                     depotPaths = sorted(settings['depot-paths'])
1558                     if self.previousDepotPaths == []:
1559                         self.previousDepotPaths = depotPaths
1560                     else:
1561                         paths = []
1562                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1563                             for i in range(0, min(len(cur), len(prev))):
1564                                 if cur[i] <> prev[i]:
1565                                     i = i - 1
1566                                     break
1567
1568                             paths.append (cur[:i + 1])
1569
1570                         self.previousDepotPaths = paths
1571
1572             if p4Change > 0:
1573                 self.depotPaths = sorted(self.previousDepotPaths)
1574                 self.changeRange = "@%s,#head" % p4Change
1575                 if not self.detectBranches:
1576                     self.initialParent = parseRevision(self.branch)
1577                 if not self.silent and not self.detectBranches:
1578                     print "Performing incremental import into %s git branch" % self.branch
1579
1580         if not self.branch.startswith("refs/"):
1581             self.branch = "refs/heads/" + self.branch
1582
1583         if len(args) == 0 and self.depotPaths:
1584             if not self.silent:
1585                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1586         else:
1587             if self.depotPaths and self.depotPaths != args:
1588                 print ("previous import used depot path %s and now %s was specified. "
1589                        "This doesn't work!" % (' '.join (self.depotPaths),
1590                                                ' '.join (args)))
1591                 sys.exit(1)
1592
1593             self.depotPaths = sorted(args)
1594
1595         revision = ""
1596         self.users = {}
1597
1598         newPaths = []
1599         for p in self.depotPaths:
1600             if p.find("@") != -1:
1601                 atIdx = p.index("@")
1602                 self.changeRange = p[atIdx:]
1603                 if self.changeRange == "@all":
1604                     self.changeRange = ""
1605                 elif ',' not in self.changeRange:
1606                     revision = self.changeRange
1607                     self.changeRange = ""
1608                 p = p[:atIdx]
1609             elif p.find("#") != -1:
1610                 hashIdx = p.index("#")
1611                 revision = p[hashIdx:]
1612                 p = p[:hashIdx]
1613             elif self.previousDepotPaths == []:
1614                 revision = "#head"
1615
1616             p = re.sub ("\.\.\.$", "", p)
1617             if not p.endswith("/"):
1618                 p += "/"
1619
1620             newPaths.append(p)
1621
1622         self.depotPaths = newPaths
1623
1624
1625         self.loadUserMapFromCache()
1626         self.labels = {}
1627         if self.detectLabels:
1628             self.getLabels();
1629
1630         if self.detectBranches:
1631             ## FIXME - what's a P4 projectName ?
1632             self.projectName = self.guessProjectName()
1633
1634             if self.hasOrigin:
1635                 self.getBranchMappingFromGitBranches()
1636             else:
1637                 self.getBranchMapping()
1638             if self.verbose:
1639                 print "p4-git branches: %s" % self.p4BranchesInGit
1640                 print "initial parents: %s" % self.initialParents
1641             for b in self.p4BranchesInGit:
1642                 if b != "master":
1643
1644                     ## FIXME
1645                     b = b[len(self.projectName):]
1646                 self.createdBranches.add(b)
1647
1648         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1649
1650         importProcess = subprocess.Popen(["git", "fast-import"],
1651                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1652                                          stderr=subprocess.PIPE);
1653         self.gitOutput = importProcess.stdout
1654         self.gitStream = importProcess.stdin
1655         self.gitError = importProcess.stderr
1656
1657         if revision:
1658             self.importHeadRevision(revision)
1659         else:
1660             changes = []
1661
1662             if len(self.changesFile) > 0:
1663                 output = open(self.changesFile).readlines()
1664                 changeSet = set()
1665                 for line in output:
1666                     changeSet.add(int(line))
1667
1668                 for change in changeSet:
1669                     changes.append(change)
1670
1671                 changes.sort()
1672             else:
1673                 if self.verbose:
1674                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1675                                                               self.changeRange)
1676                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1677
1678                 if len(self.maxChanges) > 0:
1679                     changes = changes[:min(int(self.maxChanges), len(changes))]
1680
1681             if len(changes) == 0:
1682                 if not self.silent:
1683                     print "No changes to import!"
1684                 return True
1685
1686             if not self.silent and not self.detectBranches:
1687                 print "Import destination: %s" % self.branch
1688
1689             self.updatedBranches = set()
1690
1691             self.importChanges(changes)
1692
1693             if not self.silent:
1694                 print ""
1695                 if len(self.updatedBranches) > 0:
1696                     sys.stdout.write("Updated branches: ")
1697                     for b in self.updatedBranches:
1698                         sys.stdout.write("%s " % b)
1699                     sys.stdout.write("\n")
1700
1701         self.gitStream.close()
1702         if importProcess.wait() != 0:
1703             die("fast-import failed: %s" % self.gitError.read())
1704         self.gitOutput.close()
1705         self.gitError.close()
1706
1707         return True
1708
1709 class P4Rebase(Command):
1710     def __init__(self):
1711         Command.__init__(self)
1712         self.options = [ ]
1713         self.description = ("Fetches the latest revision from perforce and "
1714                             + "rebases the current work (branch) against it")
1715         self.verbose = False
1716
1717     def run(self, args):
1718         sync = P4Sync()
1719         sync.run([])
1720
1721         return self.rebase()
1722
1723     def rebase(self):
1724         if os.system("git update-index --refresh") != 0:
1725             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1726         if len(read_pipe("git diff-index HEAD --")) > 0:
1727             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1728
1729         [upstream, settings] = findUpstreamBranchPoint()
1730         if len(upstream) == 0:
1731             die("Cannot find upstream branchpoint for rebase")
1732
1733         # the branchpoint may be p4/foo~3, so strip off the parent
1734         upstream = re.sub("~[0-9]+$", "", upstream)
1735
1736         print "Rebasing the current branch onto %s" % upstream
1737         oldHead = read_pipe("git rev-parse HEAD").strip()
1738         system("git rebase %s" % upstream)
1739         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1740         return True
1741
1742 class P4Clone(P4Sync):
1743     def __init__(self):
1744         P4Sync.__init__(self)
1745         self.description = "Creates a new git repository and imports from Perforce into it"
1746         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1747         self.options += [
1748             optparse.make_option("--destination", dest="cloneDestination",
1749                                  action='store', default=None,
1750                                  help="where to leave result of the clone"),
1751             optparse.make_option("-/", dest="cloneExclude",
1752                                  action="append", type="string",
1753                                  help="exclude depot path")
1754         ]
1755         self.cloneDestination = None
1756         self.needsGit = False
1757
1758     # This is required for the "append" cloneExclude action
1759     def ensure_value(self, attr, value):
1760         if not hasattr(self, attr) or getattr(self, attr) is None:
1761             setattr(self, attr, value)
1762         return getattr(self, attr)
1763
1764     def defaultDestination(self, args):
1765         ## TODO: use common prefix of args?
1766         depotPath = args[0]
1767         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1768         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1769         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1770         depotDir = re.sub(r"/$", "", depotDir)
1771         return os.path.split(depotDir)[1]
1772
1773     def run(self, args):
1774         if len(args) < 1:
1775             return False
1776
1777         if self.keepRepoPath and not self.cloneDestination:
1778             sys.stderr.write("Must specify destination for --keep-path\n")
1779             sys.exit(1)
1780
1781         depotPaths = args
1782
1783         if not self.cloneDestination and len(depotPaths) > 1:
1784             self.cloneDestination = depotPaths[-1]
1785             depotPaths = depotPaths[:-1]
1786
1787         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1788         for p in depotPaths:
1789             if not p.startswith("//"):
1790                 return False
1791
1792         if not self.cloneDestination:
1793             self.cloneDestination = self.defaultDestination(args)
1794
1795         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1796         if not os.path.exists(self.cloneDestination):
1797             os.makedirs(self.cloneDestination)
1798         chdir(self.cloneDestination)
1799         system("git init")
1800         self.gitdir = os.getcwd() + "/.git"
1801         if not P4Sync.run(self, depotPaths):
1802             return False
1803         if self.branch != "master":
1804             if self.importIntoRemotes:
1805                 masterbranch = "refs/remotes/p4/master"
1806             else:
1807                 masterbranch = "refs/heads/p4/master"
1808             if gitBranchExists(masterbranch):
1809                 system("git branch master %s" % masterbranch)
1810                 system("git checkout -f")
1811             else:
1812                 print "Could not detect main branch. No checkout/master branch created."
1813
1814         return True
1815
1816 class P4Branches(Command):
1817     def __init__(self):
1818         Command.__init__(self)
1819         self.options = [ ]
1820         self.description = ("Shows the git branches that hold imports and their "
1821                             + "corresponding perforce depot paths")
1822         self.verbose = False
1823
1824     def run(self, args):
1825         if originP4BranchesExist():
1826             createOrUpdateBranchesFromOrigin()
1827
1828         cmdline = "git rev-parse --symbolic "
1829         cmdline += " --remotes"
1830
1831         for line in read_pipe_lines(cmdline):
1832             line = line.strip()
1833
1834             if not line.startswith('p4/') or line == "p4/HEAD":
1835                 continue
1836             branch = line
1837
1838             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1839             settings = extractSettingsGitLog(log)
1840
1841             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1842         return True
1843
1844 class HelpFormatter(optparse.IndentedHelpFormatter):
1845     def __init__(self):
1846         optparse.IndentedHelpFormatter.__init__(self)
1847
1848     def format_description(self, description):
1849         if description:
1850             return description + "\n"
1851         else:
1852             return ""
1853
1854 def printUsage(commands):
1855     print "usage: %s <command> [options]" % sys.argv[0]
1856     print ""
1857     print "valid commands: %s" % ", ".join(commands)
1858     print ""
1859     print "Try %s <command> --help for command specific help." % sys.argv[0]
1860     print ""
1861
1862 commands = {
1863     "debug" : P4Debug,
1864     "submit" : P4Submit,
1865     "commit" : P4Submit,
1866     "sync" : P4Sync,
1867     "rebase" : P4Rebase,
1868     "clone" : P4Clone,
1869     "rollback" : P4RollBack,
1870     "branches" : P4Branches
1871 }
1872
1873
1874 def main():
1875     if len(sys.argv[1:]) == 0:
1876         printUsage(commands.keys())
1877         sys.exit(2)
1878
1879     cmd = ""
1880     cmdName = sys.argv[1]
1881     try:
1882         klass = commands[cmdName]
1883         cmd = klass()
1884     except KeyError:
1885         print "unknown command %s" % cmdName
1886         print ""
1887         printUsage(commands.keys())
1888         sys.exit(2)
1889
1890     options = cmd.options
1891     cmd.gitdir = os.environ.get("GIT_DIR", None)
1892
1893     args = sys.argv[2:]
1894
1895     if len(options) > 0:
1896         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1897
1898         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1899                                        options,
1900                                        description = cmd.description,
1901                                        formatter = HelpFormatter())
1902
1903         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1904     global verbose
1905     verbose = cmd.verbose
1906     if cmd.needsGit:
1907         if cmd.gitdir == None:
1908             cmd.gitdir = os.path.abspath(".git")
1909             if not isValidGitDir(cmd.gitdir):
1910                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1911                 if os.path.exists(cmd.gitdir):
1912                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1913                     if len(cdup) > 0:
1914                         chdir(cdup);
1915
1916         if not isValidGitDir(cmd.gitdir):
1917             if isValidGitDir(cmd.gitdir + "/.git"):
1918                 cmd.gitdir += "/.git"
1919             else:
1920                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1921
1922         os.environ["GIT_DIR"] = cmd.gitdir
1923
1924     if not cmd.run(args):
1925         parser.print_help()
1926
1927
1928 if __name__ == '__main__':
1929     main()