contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key):
 337     if not _gitConfig.has_key(key):
 338         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339     return _gitConfig[key]
 340
 341 def p4BranchesInGit(branchesAreInRemotes = True):
 342     branches = {}
 343
 344     cmdline = "git rev-parse --symbolic "
 345     if branchesAreInRemotes:
 346         cmdline += " --remotes"
 347     else:
 348         cmdline += " --branches"
 349
 350     for line in read_pipe_lines(cmdline):
 351         line = line.strip()
 352
 353         ## only import to p4/
 354         if not line.startswith('p4/') or line == "p4/HEAD":
 355             continue
 356         branch = line
 357
 358         # strip off p4
 359         branch = re.sub ("^p4/", "", line)
 360
 361         branches[branch] = parseRevision(line)
 362     return branches
 363
 364 def findUpstreamBranchPoint(head = "HEAD"):
 365     branches = p4BranchesInGit()
 366     # map from depot-path to branch name
 367     branchByDepotPath = {}
 368     for branch in branches.keys():
 369         tip = branches[branch]
 370         log = extractLogMessageFromGitCommit(tip)
 371         settings = extractSettingsGitLog(log)
 372         if settings.has_key("depot-paths"):
 373             paths = ",".join(settings["depot-paths"])
 374             branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376     settings = None
 377     parent = 0
 378     while parent < 65535:
 379         commit = head + "~%s" % parent
 380         log = extractLogMessageFromGitCommit(commit)
 381         settings = extractSettingsGitLog(log)
 382         if settings.has_key("depot-paths"):
 383             paths = ",".join(settings["depot-paths"])
 384             if branchByDepotPath.has_key(paths):
 385                 return [branchByDepotPath[paths], settings]
 386
 387         parent = parent + 1
 388
 389     return ["", settings]
 390
 391 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392     if not silent:
 393         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394                % localRefPrefix)
 395
 396     originPrefix = "origin/p4/"
 397
 398     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399         line = line.strip()
 400         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401             continue
 402
 403         headName = line[len(originPrefix):]
 404         remoteHead = localRefPrefix + headName
 405         originHead = line
 406
 407         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408         if (not original.has_key('depot-paths')
 409             or not original.has_key('change')):
 410             continue
 411
 412         update = False
 413         if not gitBranchExists(remoteHead):
 414             if verbose:
 415                 print "creating %s" % remoteHead
 416             update = True
 417         else:
 418             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419             if settings.has_key('change') > 0:
 420                 if settings['depot-paths'] == original['depot-paths']:
 421                     originP4Change = int(original['change'])
 422                     p4Change = int(settings['change'])
 423                     if originP4Change > p4Change:
 424                         print ("%s (%s) is newer than %s (%s). "
 425                                "Updating p4 branch from origin."
 426                                % (originHead, originP4Change,
 427                                   remoteHead, p4Change))
 428                         update = True
 429                 else:
 430                     print ("Ignoring: %s was imported from %s while "
 431                            "%s was imported from %s"
 432                            % (originHead, ','.join(original['depot-paths']),
 433                               remoteHead, ','.join(settings['depot-paths'])))
 434
 435         if update:
 436             system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438 def originP4BranchesExist():
 439         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441 def p4ChangesForPaths(depotPaths, changeRange):
 442     assert depotPaths
 443     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                         for p in depotPaths]))
 445
 446     changes = {}
 447     for line in output:
 448         changeNum = int(line.split(" ")[1])
 449         changes[changeNum] = True
 450
 451     changelist = changes.keys()
 452     changelist.sort()
 453     return changelist
 454
 455 class Command:
 456     def __init__(self):
 457         self.usage = "usage: %prog [options]"
 458         self.needsGit = True
 459
 460 class P4Debug(Command):
 461     def __init__(self):
 462         Command.__init__(self)
 463         self.options = [
 464             optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                  default=False),
 466             ]
 467         self.description = "A tool to debug the output of p4 -G."
 468         self.needsGit = False
 469         self.verbose = False
 470
 471     def run(self, args):
 472         j = 0
 473         for output in p4CmdList(" ".join(args)):
 474             print 'Element: %d' % j
 475             j += 1
 476             print output
 477         return True
 478
 479 class P4RollBack(Command):
 480     def __init__(self):
 481         Command.__init__(self)
 482         self.options = [
 483             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485         ]
 486         self.description = "A tool to debug the multi-branch import. Don't use :)"
 487         self.verbose = False
 488         self.rollbackLocalBranches = False
 489
 490     def run(self, args):
 491         if len(args) != 1:
 492             return False
 493         maxChange = int(args[0])
 494
 495         if "p4ExitCode" in p4Cmd("changes -m 1"):
 496             die("Problems executing p4");
 497
 498         if self.rollbackLocalBranches:
 499             refPrefix = "refs/heads/"
 500             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501         else:
 502             refPrefix = "refs/remotes/"
 503             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505         for line in lines:
 506             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                 line = line.strip()
 508                 ref = refPrefix + line
 509                 log = extractLogMessageFromGitCommit(ref)
 510                 settings = extractSettingsGitLog(log)
 511
 512                 depotPaths = settings['depot-paths']
 513                 change = settings['change']
 514
 515                 changed = False
 516
 517                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                            for p in depotPaths]))) == 0:
 519                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                     continue
 522
 523                 while change and int(change) > maxChange:
 524                     changed = True
 525                     if self.verbose:
 526                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                     system("git update-ref %s \"%s^\"" % (ref, ref))
 528                     log = extractLogMessageFromGitCommit(ref)
 529                     settings =  extractSettingsGitLog(log)
 530
 531
 532                     depotPaths = settings['depot-paths']
 533                     change = settings['change']
 534
 535                 if changed:
 536                     print "%s rewound to %s" % (ref, change)
 537
 538         return True
 539
 540 class P4Submit(Command):
 541     def __init__(self):
 542         Command.__init__(self)
 543         self.options = [
 544                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                 optparse.make_option("--origin", dest="origin"),
 546                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 547         ]
 548         self.description = "Submit changes from git to the perforce depot."
 549         self.usage += " [name of git branch to submit into perforce depot]"
 550         self.interactive = True
 551         self.origin = ""
 552         self.detectRename = False
 553         self.verbose = False
 554         self.isWindows = (platform.system() == "Windows")
 555
 556     def check(self):
 557         if len(p4CmdList("opened ...")) > 0:
 558             die("You have files opened with perforce! Close them before starting the sync.")
 559
 560     # replaces everything between 'Description:' and the next P4 submit template field with the
 561     # commit message
 562     def prepareLogMessage(self, template, message):
 563         result = ""
 564
 565         inDescriptionSection = False
 566
 567         for line in template.split("\n"):
 568             if line.startswith("#"):
 569                 result += line + "\n"
 570                 continue
 571
 572             if inDescriptionSection:
 573                 if line.startswith("Files:"):
 574                     inDescriptionSection = False
 575                 else:
 576                     continue
 577             else:
 578                 if line.startswith("Description:"):
 579                     inDescriptionSection = True
 580                     line += "\n"
 581                     for messageLine in message.split("\n"):
 582                         line += "\t" + messageLine + "\n"
 583
 584             result += line + "\n"
 585
 586         return result
 587
 588     def prepareSubmitTemplate(self):
 589         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590         template = ""
 591         inFilesSection = False
 592         for line in p4_read_pipe_lines("change -o"):
 593             if line.endswith("\r\n"):
 594                 line = line[:-2] + "\n"
 595             if inFilesSection:
 596                 if line.startswith("\t"):
 597                     # path starts and ends with a tab
 598                     path = line[1:]
 599                     lastTab = path.rfind("\t")
 600                     if lastTab != -1:
 601                         path = path[:lastTab]
 602                         if not path.startswith(self.depotPath):
 603                             continue
 604                 else:
 605                     inFilesSection = False
 606             else:
 607                 if line.startswith("Files:"):
 608                     inFilesSection = True
 609
 610             template += line
 611
 612         return template
 613
 614     def applyCommit(self, id):
 615         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616         diffOpts = ("", "-M")[self.detectRename]
 617         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618         filesToAdd = set()
 619         filesToDelete = set()
 620         editedFiles = set()
 621         filesToChangeExecBit = {}
 622         for line in diff:
 623             diff = parseDiffTreeEntry(line)
 624             modifier = diff['status']
 625             path = diff['src']
 626             if modifier == "M":
 627                 p4_system("edit \"%s\"" % path)
 628                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                     filesToChangeExecBit[path] = diff['dst_mode']
 630                 editedFiles.add(path)
 631             elif modifier == "A":
 632                 filesToAdd.add(path)
 633                 filesToChangeExecBit[path] = diff['dst_mode']
 634                 if path in filesToDelete:
 635                     filesToDelete.remove(path)
 636             elif modifier == "D":
 637                 filesToDelete.add(path)
 638                 if path in filesToAdd:
 639                     filesToAdd.remove(path)
 640             elif modifier == "R":
 641                 src, dest = diff['src'], diff['dst']
 642                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                 p4_system("edit \"%s\"" % (dest))
 644                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                     filesToChangeExecBit[dest] = diff['dst_mode']
 646                 os.unlink(dest)
 647                 editedFiles.add(dest)
 648                 filesToDelete.add(src)
 649             else:
 650                 die("unknown modifier %s for %s" % (modifier, path))
 651
 652         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653         patchcmd = diffcmd + " | git apply "
 654         tryPatchCmd = patchcmd + "--check -"
 655         applyPatchCmd = patchcmd + "--check --apply -"
 656
 657         if os.system(tryPatchCmd) != 0:
 658             print "Unfortunately applying the change failed!"
 659             print "What do you want to do?"
 660             response = "x"
 661             while response != "s" and response != "a" and response != "w":
 662                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664             if response == "s":
 665                 print "Skipping! Good luck with the next patches..."
 666                 for f in editedFiles:
 667                     p4_system("revert \"%s\"" % f);
 668                 for f in filesToAdd:
 669                     system("rm %s" %f)
 670                 return
 671             elif response == "a":
 672                 os.system(applyPatchCmd)
 673                 if len(filesToAdd) > 0:
 674                     print "You may also want to call p4 add on the following files:"
 675                     print " ".join(filesToAdd)
 676                 if len(filesToDelete):
 677                     print "The following files should be scheduled for deletion with p4 delete:"
 678                     print " ".join(filesToDelete)
 679                 die("Please resolve and submit the conflict manually and "
 680                     + "continue afterwards with git-p4 submit --continue")
 681             elif response == "w":
 682                 system(diffcmd + " > patch.txt")
 683                 print "Patch saved to patch.txt in %s !" % self.clientPath
 684                 die("Please resolve and submit the conflict manually and "
 685                     "continue afterwards with git-p4 submit --continue")
 686
 687         system(applyPatchCmd)
 688
 689         for f in filesToAdd:
 690             p4_system("add \"%s\"" % f)
 691         for f in filesToDelete:
 692             p4_system("revert \"%s\"" % f)
 693             p4_system("delete \"%s\"" % f)
 694
 695         # Set/clear executable bits
 696         for f in filesToChangeExecBit.keys():
 697             mode = filesToChangeExecBit[f]
 698             setP4ExecBit(f, mode)
 699
 700         logMessage = extractLogMessageFromGitCommit(id)
 701         logMessage = logMessage.strip()
 702
 703         template = self.prepareSubmitTemplate()
 704
 705         if self.interactive:
 706             submitTemplate = self.prepareLogMessage(template, logMessage)
 707             if os.environ.has_key("P4DIFF"):
 708                 del(os.environ["P4DIFF"])
 709             diff = ""
 710             for editedFile in editedFiles:
 711                 diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713             newdiff = ""
 714             for newFile in filesToAdd:
 715                 newdiff += "==== new file ====\n"
 716                 newdiff += "--- /dev/null\n"
 717                 newdiff += "+++ %s\n" % newFile
 718                 f = open(newFile, "r")
 719                 for line in f.readlines():
 720                     newdiff += "+" + line
 721                 f.close()
 722
 723             separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725             [handle, fileName] = tempfile.mkstemp()
 726             tmpFile = os.fdopen(handle, "w+")
 727             if self.isWindows:
 728                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                 separatorLine = separatorLine.replace("\n", "\r\n")
 730                 newdiff = newdiff.replace("\n", "\r\n")
 731             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732             tmpFile.close()
 733             mtime = os.stat(fileName).st_mtime
 734             if os.environ.has_key("P4EDITOR"):
 735                 editor = os.environ.get("P4EDITOR")
 736             else:
 737                 editor = read_pipe("git var GIT_EDITOR").strip()
 738             system(editor + " " + fileName)
 739
 740             response = "y"
 741             if os.stat(fileName).st_mtime <= mtime:
 742                 response = "x"
 743                 while response != "y" and response != "n":
 744                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746             if response == "y":
 747                 tmpFile = open(fileName, "rb")
 748                 message = tmpFile.read()
 749                 tmpFile.close()
 750                 submitTemplate = message[:message.index(separatorLine)]
 751                 if self.isWindows:
 752                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                 p4_write_pipe("submit -i", submitTemplate)
 754             else:
 755                 for f in editedFiles:
 756                     p4_system("revert \"%s\"" % f);
 757                 for f in filesToAdd:
 758                     p4_system("revert \"%s\"" % f);
 759                     system("rm %s" %f)
 760
 761             os.remove(fileName)
 762         else:
 763             fileName = "submit.txt"
 764             file = open(fileName, "w+")
 765             file.write(self.prepareLogMessage(template, logMessage))
 766             file.close()
 767             print ("Perforce submit template written as %s. "
 768                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                    % (fileName, fileName))
 770
 771     def run(self, args):
 772         if len(args) == 0:
 773             self.master = currentGitBranch()
 774             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                 die("Detecting current git branch failed!")
 776         elif len(args) == 1:
 777             self.master = args[0]
 778         else:
 779             return False
 780
 781         allowSubmit = gitConfig("git-p4.allowSubmit")
 782         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783             die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785         [upstream, settings] = findUpstreamBranchPoint()
 786         self.depotPath = settings['depot-paths'][0]
 787         if len(self.origin) == 0:
 788             self.origin = upstream
 789
 790         if self.verbose:
 791             print "Origin branch is " + self.origin
 792
 793         if len(self.depotPath) == 0:
 794             print "Internal error: cannot locate perforce depot path from existing branches"
 795             sys.exit(128)
 796
 797         self.clientPath = p4Where(self.depotPath)
 798
 799         if len(self.clientPath) == 0:
 800             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801             sys.exit(128)
 802
 803         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804         self.oldWorkingDirectory = os.getcwd()
 805
 806         chdir(self.clientPath)
 807         print "Synchronizing p4 checkout..."
 808         p4_system("sync ...")
 809
 810         self.check()
 811
 812         commits = []
 813         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814             commits.append(line.strip())
 815         commits.reverse()
 816
 817         while len(commits) > 0:
 818             commit = commits[0]
 819             commits = commits[1:]
 820             self.applyCommit(commit)
 821             if not self.interactive:
 822                 break
 823
 824         if len(commits) == 0:
 825             print "All changes applied!"
 826             chdir(self.oldWorkingDirectory)
 827
 828             sync = P4Sync()
 829             sync.run([])
 830
 831             rebase = P4Rebase()
 832             rebase.rebase()
 833
 834         return True
 835
 836 class P4Sync(Command):
 837     delete_actions = ( "delete", "move/delete", "purge" )
 838
 839     def __init__(self):
 840         Command.__init__(self)
 841         self.options = [
 842                 optparse.make_option("--branch", dest="branch"),
 843                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 844                 optparse.make_option("--changesfile", dest="changesFile"),
 845                 optparse.make_option("--silent", dest="silent", action="store_true"),
 846                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 847                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 848                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 849                                      help="Import into refs/heads/ , not refs/remotes"),
 850                 optparse.make_option("--max-changes", dest="maxChanges"),
 851                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 852                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 853                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 854                                      help="Only sync files that are included in the Perforce Client Spec")
 855         ]
 856         self.description = """Imports from Perforce into a git repository.\n
 857     example:
 858     //depot/my/project/ -- to import the current head
 859     //depot/my/project/@all -- to import everything
 860     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 861
 862     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 863
 864         self.usage += " //depot/path[@revRange]"
 865         self.silent = False
 866         self.createdBranches = set()
 867         self.committedChanges = set()
 868         self.branch = ""
 869         self.detectBranches = False
 870         self.detectLabels = False
 871         self.changesFile = ""
 872         self.syncWithOrigin = True
 873         self.verbose = False
 874         self.importIntoRemotes = True
 875         self.maxChanges = ""
 876         self.isWindows = (platform.system() == "Windows")
 877         self.keepRepoPath = False
 878         self.depotPaths = None
 879         self.p4BranchesInGit = []
 880         self.cloneExclude = []
 881         self.useClientSpec = False
 882         self.clientSpecDirs = []
 883
 884         if gitConfig("git-p4.syncFromOrigin") == "false":
 885             self.syncWithOrigin = False
 886
 887     def extractFilesFromCommit(self, commit):
 888         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 889                              for path in self.cloneExclude]
 890         files = []
 891         fnum = 0
 892         while commit.has_key("depotFile%s" % fnum):
 893             path =  commit["depotFile%s" % fnum]
 894
 895             if [p for p in self.cloneExclude
 896                 if path.startswith (p)]:
 897                 found = False
 898             else:
 899                 found = [p for p in self.depotPaths
 900                          if path.startswith (p)]
 901             if not found:
 902                 fnum = fnum + 1
 903                 continue
 904
 905             file = {}
 906             file["path"] = path
 907             file["rev"] = commit["rev%s" % fnum]
 908             file["action"] = commit["action%s" % fnum]
 909             file["type"] = commit["type%s" % fnum]
 910             files.append(file)
 911             fnum = fnum + 1
 912         return files
 913
 914     def stripRepoPath(self, path, prefixes):
 915         if self.keepRepoPath:
 916             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 917
 918         for p in prefixes:
 919             if path.startswith(p):
 920                 path = path[len(p):]
 921
 922         return path
 923
 924     def splitFilesIntoBranches(self, commit):
 925         branches = {}
 926         fnum = 0
 927         while commit.has_key("depotFile%s" % fnum):
 928             path =  commit["depotFile%s" % fnum]
 929             found = [p for p in self.depotPaths
 930                      if path.startswith (p)]
 931             if not found:
 932                 fnum = fnum + 1
 933                 continue
 934
 935             file = {}
 936             file["path"] = path
 937             file["rev"] = commit["rev%s" % fnum]
 938             file["action"] = commit["action%s" % fnum]
 939             file["type"] = commit["type%s" % fnum]
 940             fnum = fnum + 1
 941
 942             relPath = self.stripRepoPath(path, self.depotPaths)
 943
 944             for branch in self.knownBranches.keys():
 945
 946                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 947                 if relPath.startswith(branch + "/"):
 948                     if branch not in branches:
 949                         branches[branch] = []
 950                     branches[branch].append(file)
 951                     break
 952
 953         return branches
 954
 955     # output one file from the P4 stream
 956     # - helper for streamP4Files
 957
 958     def streamOneP4File(self, file, contents):
 959         if file["type"] == "apple":
 960             print "\nfile %s is a strange apple file that forks. Ignoring" % \
 961                 file['depotFile']
 962             return
 963
 964         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 965         if verbose:
 966             sys.stderr.write("%s\n" % relPath)
 967
 968         mode = "644"
 969         if isP4Exec(file["type"]):
 970             mode = "755"
 971         elif file["type"] == "symlink":
 972             mode = "120000"
 973             # p4 print on a symlink contains "target\n", so strip it off
 974             data = ''.join(contents)
 975             contents = [data[:-1]]
 976
 977         if self.isWindows and file["type"].endswith("text"):
 978             mangled = []
 979             for data in contents:
 980                 data = data.replace("\r\n", "\n")
 981                 mangled.append(data)
 982             contents = mangled
 983
 984         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 985             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 986         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 987             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 988
 989         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 990
 991         # total length...
 992         length = 0
 993         for d in contents:
 994             length = length + len(d)
 995
 996         self.gitStream.write("data %d\n" % length)
 997         for d in contents:
 998             self.gitStream.write(d)
 999         self.gitStream.write("\n")
1000
1001     def streamOneP4Deletion(self, file):
1002         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1003         if verbose:
1004             sys.stderr.write("delete %s\n" % relPath)
1005         self.gitStream.write("D %s\n" % relPath)
1006
1007     # handle another chunk of streaming data
1008     def streamP4FilesCb(self, marshalled):
1009
1010         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1011             # start of a new file - output the old one first
1012             self.streamOneP4File(self.stream_file, self.stream_contents)
1013             self.stream_file = {}
1014             self.stream_contents = []
1015             self.stream_have_file_info = False
1016
1017         # pick up the new file information... for the
1018         # 'data' field we need to append to our array
1019         for k in marshalled.keys():
1020             if k == 'data':
1021                 self.stream_contents.append(marshalled['data'])
1022             else:
1023                 self.stream_file[k] = marshalled[k]
1024
1025         self.stream_have_file_info = True
1026
1027     # Stream directly from "p4 files" into "git fast-import"
1028     def streamP4Files(self, files):
1029         filesForCommit = []
1030         filesToRead = []
1031         filesToDelete = []
1032
1033         for f in files:
1034             includeFile = True
1035             for val in self.clientSpecDirs:
1036                 if f['path'].startswith(val[0]):
1037                     if val[1] <= 0:
1038                         includeFile = False
1039                     break
1040
1041             if includeFile:
1042                 filesForCommit.append(f)
1043                 if f['action'] in self.delete_actions:
1044                     filesToDelete.append(f)
1045                 else:
1046                     filesToRead.append(f)
1047
1048         # deleted files...
1049         for f in filesToDelete:
1050             self.streamOneP4Deletion(f)
1051
1052         if len(filesToRead) > 0:
1053             self.stream_file = {}
1054             self.stream_contents = []
1055             self.stream_have_file_info = False
1056
1057             # curry self argument
1058             def streamP4FilesCbSelf(entry):
1059                 self.streamP4FilesCb(entry)
1060
1061             p4CmdList("-x - print",
1062                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1063                                                   for f in filesToRead]),
1064                 cb=streamP4FilesCbSelf)
1065
1066             # do the last chunk
1067             if self.stream_file.has_key('depotFile'):
1068                 self.streamOneP4File(self.stream_file, self.stream_contents)
1069
1070     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1071         epoch = details["time"]
1072         author = details["user"]
1073         self.branchPrefixes = branchPrefixes
1074
1075         if self.verbose:
1076             print "commit into %s" % branch
1077
1078         # start with reading files; if that fails, we should not
1079         # create a commit.
1080         new_files = []
1081         for f in files:
1082             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1083                 new_files.append (f)
1084             else:
1085                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1086
1087         self.gitStream.write("commit %s\n" % branch)
1088 #        gitStream.write("mark :%s\n" % details["change"])
1089         self.committedChanges.add(int(details["change"]))
1090         committer = ""
1091         if author not in self.users:
1092             self.getUserMapFromPerforceServer()
1093         if author in self.users:
1094             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1095         else:
1096             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1097
1098         self.gitStream.write("committer %s\n" % committer)
1099
1100         self.gitStream.write("data <<EOT\n")
1101         self.gitStream.write(details["desc"])
1102         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1103                              % (','.join (branchPrefixes), details["change"]))
1104         if len(details['options']) > 0:
1105             self.gitStream.write(": options = %s" % details['options'])
1106         self.gitStream.write("]\nEOT\n\n")
1107
1108         if len(parent) > 0:
1109             if self.verbose:
1110                 print "parent %s" % parent
1111             self.gitStream.write("from %s\n" % parent)
1112
1113         self.streamP4Files(new_files)
1114         self.gitStream.write("\n")
1115
1116         change = int(details["change"])
1117
1118         if self.labels.has_key(change):
1119             label = self.labels[change]
1120             labelDetails = label[0]
1121             labelRevisions = label[1]
1122             if self.verbose:
1123                 print "Change %s is labelled %s" % (change, labelDetails)
1124
1125             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1126                                                     for p in branchPrefixes]))
1127
1128             if len(files) == len(labelRevisions):
1129
1130                 cleanedFiles = {}
1131                 for info in files:
1132                     if info["action"] in self.delete_actions:
1133                         continue
1134                     cleanedFiles[info["depotFile"]] = info["rev"]
1135
1136                 if cleanedFiles == labelRevisions:
1137                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1138                     self.gitStream.write("from %s\n" % branch)
1139
1140                     owner = labelDetails["Owner"]
1141                     tagger = ""
1142                     if author in self.users:
1143                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1144                     else:
1145                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1146                     self.gitStream.write("tagger %s\n" % tagger)
1147                     self.gitStream.write("data <<EOT\n")
1148                     self.gitStream.write(labelDetails["Description"])
1149                     self.gitStream.write("EOT\n\n")
1150
1151                 else:
1152                     if not self.silent:
1153                         print ("Tag %s does not match with change %s: files do not match."
1154                                % (labelDetails["label"], change))
1155
1156             else:
1157                 if not self.silent:
1158                     print ("Tag %s does not match with change %s: file count is different."
1159                            % (labelDetails["label"], change))
1160
1161     def getUserCacheFilename(self):
1162         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1163         return home + "/.gitp4-usercache.txt"
1164
1165     def getUserMapFromPerforceServer(self):
1166         if self.userMapFromPerforceServer:
1167             return
1168         self.users = {}
1169
1170         for output in p4CmdList("users"):
1171             if not output.has_key("User"):
1172                 continue
1173             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1174
1175
1176         s = ''
1177         for (key, val) in self.users.items():
1178             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1179
1180         open(self.getUserCacheFilename(), "wb").write(s)
1181         self.userMapFromPerforceServer = True
1182
1183     def loadUserMapFromCache(self):
1184         self.users = {}
1185         self.userMapFromPerforceServer = False
1186         try:
1187             cache = open(self.getUserCacheFilename(), "rb")
1188             lines = cache.readlines()
1189             cache.close()
1190             for line in lines:
1191                 entry = line.strip().split("\t")
1192                 self.users[entry[0]] = entry[1]
1193         except IOError:
1194             self.getUserMapFromPerforceServer()
1195
1196     def getLabels(self):
1197         self.labels = {}
1198
1199         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1200         if len(l) > 0 and not self.silent:
1201             print "Finding files belonging to labels in %s" % `self.depotPaths`
1202
1203         for output in l:
1204             label = output["label"]
1205             revisions = {}
1206             newestChange = 0
1207             if self.verbose:
1208                 print "Querying files for label %s" % label
1209             for file in p4CmdList("files "
1210                                   +  ' '.join (["%s...@%s" % (p, label)
1211                                                 for p in self.depotPaths])):
1212                 revisions[file["depotFile"]] = file["rev"]
1213                 change = int(file["change"])
1214                 if change > newestChange:
1215                     newestChange = change
1216
1217             self.labels[newestChange] = [output, revisions]
1218
1219         if self.verbose:
1220             print "Label changes: %s" % self.labels.keys()
1221
1222     def guessProjectName(self):
1223         for p in self.depotPaths:
1224             if p.endswith("/"):
1225                 p = p[:-1]
1226             p = p[p.strip().rfind("/") + 1:]
1227             if not p.endswith("/"):
1228                p += "/"
1229             return p
1230
1231     def getBranchMapping(self):
1232         lostAndFoundBranches = set()
1233
1234         for info in p4CmdList("branches"):
1235             details = p4Cmd("branch -o %s" % info["branch"])
1236             viewIdx = 0
1237             while details.has_key("View%s" % viewIdx):
1238                 paths = details["View%s" % viewIdx].split(" ")
1239                 viewIdx = viewIdx + 1
1240                 # require standard //depot/foo/... //depot/bar/... mapping
1241                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1242                     continue
1243                 source = paths[0]
1244                 destination = paths[1]
1245                 ## HACK
1246                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1247                     source = source[len(self.depotPaths[0]):-4]
1248                     destination = destination[len(self.depotPaths[0]):-4]
1249
1250                     if destination in self.knownBranches:
1251                         if not self.silent:
1252                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1253                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1254                         continue
1255
1256                     self.knownBranches[destination] = source
1257
1258                     lostAndFoundBranches.discard(destination)
1259
1260                     if source not in self.knownBranches:
1261                         lostAndFoundBranches.add(source)
1262
1263
1264         for branch in lostAndFoundBranches:
1265             self.knownBranches[branch] = branch
1266
1267     def getBranchMappingFromGitBranches(self):
1268         branches = p4BranchesInGit(self.importIntoRemotes)
1269         for branch in branches.keys():
1270             if branch == "master":
1271                 branch = "main"
1272             else:
1273                 branch = branch[len(self.projectName):]
1274             self.knownBranches[branch] = branch
1275
1276     def listExistingP4GitBranches(self):
1277         # branches holds mapping from name to commit
1278         branches = p4BranchesInGit(self.importIntoRemotes)
1279         self.p4BranchesInGit = branches.keys()
1280         for branch in branches.keys():
1281             self.initialParents[self.refPrefix + branch] = branches[branch]
1282
1283     def updateOptionDict(self, d):
1284         option_keys = {}
1285         if self.keepRepoPath:
1286             option_keys['keepRepoPath'] = 1
1287
1288         d["options"] = ' '.join(sorted(option_keys.keys()))
1289
1290     def readOptions(self, d):
1291         self.keepRepoPath = (d.has_key('options')
1292                              and ('keepRepoPath' in d['options']))
1293
1294     def gitRefForBranch(self, branch):
1295         if branch == "main":
1296             return self.refPrefix + "master"
1297
1298         if len(branch) <= 0:
1299             return branch
1300
1301         return self.refPrefix + self.projectName + branch
1302
1303     def gitCommitByP4Change(self, ref, change):
1304         if self.verbose:
1305             print "looking in ref " + ref + " for change %s using bisect..." % change
1306
1307         earliestCommit = ""
1308         latestCommit = parseRevision(ref)
1309
1310         while True:
1311             if self.verbose:
1312                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1313             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1314             if len(next) == 0:
1315                 if self.verbose:
1316                     print "argh"
1317                 return ""
1318             log = extractLogMessageFromGitCommit(next)
1319             settings = extractSettingsGitLog(log)
1320             currentChange = int(settings['change'])
1321             if self.verbose:
1322                 print "current change %s" % currentChange
1323
1324             if currentChange == change:
1325                 if self.verbose:
1326                     print "found %s" % next
1327                 return next
1328
1329             if currentChange < change:
1330                 earliestCommit = "^%s" % next
1331             else:
1332                 latestCommit = "%s" % next
1333
1334         return ""
1335
1336     def importNewBranch(self, branch, maxChange):
1337         # make fast-import flush all changes to disk and update the refs using the checkpoint
1338         # command so that we can try to find the branch parent in the git history
1339         self.gitStream.write("checkpoint\n\n");
1340         self.gitStream.flush();
1341         branchPrefix = self.depotPaths[0] + branch + "/"
1342         range = "@1,%s" % maxChange
1343         #print "prefix" + branchPrefix
1344         changes = p4ChangesForPaths([branchPrefix], range)
1345         if len(changes) <= 0:
1346             return False
1347         firstChange = changes[0]
1348         #print "first change in branch: %s" % firstChange
1349         sourceBranch = self.knownBranches[branch]
1350         sourceDepotPath = self.depotPaths[0] + sourceBranch
1351         sourceRef = self.gitRefForBranch(sourceBranch)
1352         #print "source " + sourceBranch
1353
1354         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1355         #print "branch parent: %s" % branchParentChange
1356         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1357         if len(gitParent) > 0:
1358             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1359             #print "parent git commit: %s" % gitParent
1360
1361         self.importChanges(changes)
1362         return True
1363
1364     def importChanges(self, changes):
1365         cnt = 1
1366         for change in changes:
1367             description = p4Cmd("describe %s" % change)
1368             self.updateOptionDict(description)
1369
1370             if not self.silent:
1371                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1372                 sys.stdout.flush()
1373             cnt = cnt + 1
1374
1375             try:
1376                 if self.detectBranches:
1377                     branches = self.splitFilesIntoBranches(description)
1378                     for branch in branches.keys():
1379                         ## HACK  --hwn
1380                         branchPrefix = self.depotPaths[0] + branch + "/"
1381
1382                         parent = ""
1383
1384                         filesForCommit = branches[branch]
1385
1386                         if self.verbose:
1387                             print "branch is %s" % branch
1388
1389                         self.updatedBranches.add(branch)
1390
1391                         if branch not in self.createdBranches:
1392                             self.createdBranches.add(branch)
1393                             parent = self.knownBranches[branch]
1394                             if parent == branch:
1395                                 parent = ""
1396                             else:
1397                                 fullBranch = self.projectName + branch
1398                                 if fullBranch not in self.p4BranchesInGit:
1399                                     if not self.silent:
1400                                         print("\n    Importing new branch %s" % fullBranch);
1401                                     if self.importNewBranch(branch, change - 1):
1402                                         parent = ""
1403                                         self.p4BranchesInGit.append(fullBranch)
1404                                     if not self.silent:
1405                                         print("\n    Resuming with change %s" % change);
1406
1407                                 if self.verbose:
1408                                     print "parent determined through known branches: %s" % parent
1409
1410                         branch = self.gitRefForBranch(branch)
1411                         parent = self.gitRefForBranch(parent)
1412
1413                         if self.verbose:
1414                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1415
1416                         if len(parent) == 0 and branch in self.initialParents:
1417                             parent = self.initialParents[branch]
1418                             del self.initialParents[branch]
1419
1420                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1421                 else:
1422                     files = self.extractFilesFromCommit(description)
1423                     self.commit(description, files, self.branch, self.depotPaths,
1424                                 self.initialParent)
1425                     self.initialParent = ""
1426             except IOError:
1427                 print self.gitError.read()
1428                 sys.exit(1)
1429
1430     def importHeadRevision(self, revision):
1431         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1432
1433         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1434         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1435                            % (' '.join(self.depotPaths), revision))
1436         details["change"] = revision
1437         newestRevision = 0
1438
1439         fileCnt = 0
1440         for info in p4CmdList("files "
1441                               +  ' '.join(["%s...%s"
1442                                            % (p, revision)
1443                                            for p in self.depotPaths])):
1444
1445             if 'code' in info and info['code'] == 'error':
1446                 sys.stderr.write("p4 returned an error: %s\n"
1447                                  % info['data'])
1448                 sys.exit(1)
1449             if 'p4ExitCode' in info:
1450                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1451                 sys.exit(1)
1452
1453
1454             change = int(info["change"])
1455             if change > newestRevision:
1456                 newestRevision = change
1457
1458             if info["action"] in self.delete_actions:
1459                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1460                 #fileCnt = fileCnt + 1
1461                 continue
1462
1463             for prop in ["depotFile", "rev", "action", "type" ]:
1464                 details["%s%s" % (prop, fileCnt)] = info[prop]
1465
1466             fileCnt = fileCnt + 1
1467
1468         details["change"] = newestRevision
1469         self.updateOptionDict(details)
1470         try:
1471             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1472         except IOError:
1473             print "IO error with git fast-import. Is your git version recent enough?"
1474             print self.gitError.read()
1475
1476
1477     def getClientSpec(self):
1478         specList = p4CmdList( "client -o" )
1479         temp = {}
1480         for entry in specList:
1481             for k,v in entry.iteritems():
1482                 if k.startswith("View"):
1483                     if v.startswith('"'):
1484                         start = 1
1485                     else:
1486                         start = 0
1487                     index = v.find("...")
1488                     v = v[start:index]
1489                     if v.startswith("-"):
1490                         v = v[1:]
1491                         temp[v] = -len(v)
1492                     else:
1493                         temp[v] = len(v)
1494         self.clientSpecDirs = temp.items()
1495         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1496
1497     def run(self, args):
1498         self.depotPaths = []
1499         self.changeRange = ""
1500         self.initialParent = ""
1501         self.previousDepotPaths = []
1502
1503         # map from branch depot path to parent branch
1504         self.knownBranches = {}
1505         self.initialParents = {}
1506         self.hasOrigin = originP4BranchesExist()
1507         if not self.syncWithOrigin:
1508             self.hasOrigin = False
1509
1510         if self.importIntoRemotes:
1511             self.refPrefix = "refs/remotes/p4/"
1512         else:
1513             self.refPrefix = "refs/heads/p4/"
1514
1515         if self.syncWithOrigin and self.hasOrigin:
1516             if not self.silent:
1517                 print "Syncing with origin first by calling git fetch origin"
1518             system("git fetch origin")
1519
1520         if len(self.branch) == 0:
1521             self.branch = self.refPrefix + "master"
1522             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1523                 system("git update-ref %s refs/heads/p4" % self.branch)
1524                 system("git branch -D p4");
1525             # create it /after/ importing, when master exists
1526             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1527                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1528
1529         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1530             self.getClientSpec()
1531
1532         # TODO: should always look at previous commits,
1533         # merge with previous imports, if possible.
1534         if args == []:
1535             if self.hasOrigin:
1536                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1537             self.listExistingP4GitBranches()
1538
1539             if len(self.p4BranchesInGit) > 1:
1540                 if not self.silent:
1541                     print "Importing from/into multiple branches"
1542                 self.detectBranches = True
1543
1544             if self.verbose:
1545                 print "branches: %s" % self.p4BranchesInGit
1546
1547             p4Change = 0
1548             for branch in self.p4BranchesInGit:
1549                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1550
1551                 settings = extractSettingsGitLog(logMsg)
1552
1553                 self.readOptions(settings)
1554                 if (settings.has_key('depot-paths')
1555                     and settings.has_key ('change')):
1556                     change = int(settings['change']) + 1
1557                     p4Change = max(p4Change, change)
1558
1559                     depotPaths = sorted(settings['depot-paths'])
1560                     if self.previousDepotPaths == []:
1561                         self.previousDepotPaths = depotPaths
1562                     else:
1563                         paths = []
1564                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1565                             for i in range(0, min(len(cur), len(prev))):
1566                                 if cur[i] <> prev[i]:
1567                                     i = i - 1
1568                                     break
1569
1570                             paths.append (cur[:i + 1])
1571
1572                         self.previousDepotPaths = paths
1573
1574             if p4Change > 0:
1575                 self.depotPaths = sorted(self.previousDepotPaths)
1576                 self.changeRange = "@%s,#head" % p4Change
1577                 if not self.detectBranches:
1578                     self.initialParent = parseRevision(self.branch)
1579                 if not self.silent and not self.detectBranches:
1580                     print "Performing incremental import into %s git branch" % self.branch
1581
1582         if not self.branch.startswith("refs/"):
1583             self.branch = "refs/heads/" + self.branch
1584
1585         if len(args) == 0 and self.depotPaths:
1586             if not self.silent:
1587                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1588         else:
1589             if self.depotPaths and self.depotPaths != args:
1590                 print ("previous import used depot path %s and now %s was specified. "
1591                        "This doesn't work!" % (' '.join (self.depotPaths),
1592                                                ' '.join (args)))
1593                 sys.exit(1)
1594
1595             self.depotPaths = sorted(args)
1596
1597         revision = ""
1598         self.users = {}
1599
1600         newPaths = []
1601         for p in self.depotPaths:
1602             if p.find("@") != -1:
1603                 atIdx = p.index("@")
1604                 self.changeRange = p[atIdx:]
1605                 if self.changeRange == "@all":
1606                     self.changeRange = ""
1607                 elif ',' not in self.changeRange:
1608                     revision = self.changeRange
1609                     self.changeRange = ""
1610                 p = p[:atIdx]
1611             elif p.find("#") != -1:
1612                 hashIdx = p.index("#")
1613                 revision = p[hashIdx:]
1614                 p = p[:hashIdx]
1615             elif self.previousDepotPaths == []:
1616                 revision = "#head"
1617
1618             p = re.sub ("\.\.\.$", "", p)
1619             if not p.endswith("/"):
1620                 p += "/"
1621
1622             newPaths.append(p)
1623
1624         self.depotPaths = newPaths
1625
1626
1627         self.loadUserMapFromCache()
1628         self.labels = {}
1629         if self.detectLabels:
1630             self.getLabels();
1631
1632         if self.detectBranches:
1633             ## FIXME - what's a P4 projectName ?
1634             self.projectName = self.guessProjectName()
1635
1636             if self.hasOrigin:
1637                 self.getBranchMappingFromGitBranches()
1638             else:
1639                 self.getBranchMapping()
1640             if self.verbose:
1641                 print "p4-git branches: %s" % self.p4BranchesInGit
1642                 print "initial parents: %s" % self.initialParents
1643             for b in self.p4BranchesInGit:
1644                 if b != "master":
1645
1646                     ## FIXME
1647                     b = b[len(self.projectName):]
1648                 self.createdBranches.add(b)
1649
1650         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1651
1652         importProcess = subprocess.Popen(["git", "fast-import"],
1653                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1654                                          stderr=subprocess.PIPE);
1655         self.gitOutput = importProcess.stdout
1656         self.gitStream = importProcess.stdin
1657         self.gitError = importProcess.stderr
1658
1659         if revision:
1660             self.importHeadRevision(revision)
1661         else:
1662             changes = []
1663
1664             if len(self.changesFile) > 0:
1665                 output = open(self.changesFile).readlines()
1666                 changeSet = set()
1667                 for line in output:
1668                     changeSet.add(int(line))
1669
1670                 for change in changeSet:
1671                     changes.append(change)
1672
1673                 changes.sort()
1674             else:
1675                 if self.verbose:
1676                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1677                                                               self.changeRange)
1678                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1679
1680                 if len(self.maxChanges) > 0:
1681                     changes = changes[:min(int(self.maxChanges), len(changes))]
1682
1683             if len(changes) == 0:
1684                 if not self.silent:
1685                     print "No changes to import!"
1686                 return True
1687
1688             if not self.silent and not self.detectBranches:
1689                 print "Import destination: %s" % self.branch
1690
1691             self.updatedBranches = set()
1692
1693             self.importChanges(changes)
1694
1695             if not self.silent:
1696                 print ""
1697                 if len(self.updatedBranches) > 0:
1698                     sys.stdout.write("Updated branches: ")
1699                     for b in self.updatedBranches:
1700                         sys.stdout.write("%s " % b)
1701                     sys.stdout.write("\n")
1702
1703         self.gitStream.close()
1704         if importProcess.wait() != 0:
1705             die("fast-import failed: %s" % self.gitError.read())
1706         self.gitOutput.close()
1707         self.gitError.close()
1708
1709         return True
1710
1711 class P4Rebase(Command):
1712     def __init__(self):
1713         Command.__init__(self)
1714         self.options = [ ]
1715         self.description = ("Fetches the latest revision from perforce and "
1716                             + "rebases the current work (branch) against it")
1717         self.verbose = False
1718
1719     def run(self, args):
1720         sync = P4Sync()
1721         sync.run([])
1722
1723         return self.rebase()
1724
1725     def rebase(self):
1726         if os.system("git update-index --refresh") != 0:
1727             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1728         if len(read_pipe("git diff-index HEAD --")) > 0:
1729             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1730
1731         [upstream, settings] = findUpstreamBranchPoint()
1732         if len(upstream) == 0:
1733             die("Cannot find upstream branchpoint for rebase")
1734
1735         # the branchpoint may be p4/foo~3, so strip off the parent
1736         upstream = re.sub("~[0-9]+$", "", upstream)
1737
1738         print "Rebasing the current branch onto %s" % upstream
1739         oldHead = read_pipe("git rev-parse HEAD").strip()
1740         system("git rebase %s" % upstream)
1741         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1742         return True
1743
1744 class P4Clone(P4Sync):
1745     def __init__(self):
1746         P4Sync.__init__(self)
1747         self.description = "Creates a new git repository and imports from Perforce into it"
1748         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1749         self.options += [
1750             optparse.make_option("--destination", dest="cloneDestination",
1751                                  action='store', default=None,
1752                                  help="where to leave result of the clone"),
1753             optparse.make_option("-/", dest="cloneExclude",
1754                                  action="append", type="string",
1755                                  help="exclude depot path")
1756         ]
1757         self.cloneDestination = None
1758         self.needsGit = False
1759
1760     # This is required for the "append" cloneExclude action
1761     def ensure_value(self, attr, value):
1762         if not hasattr(self, attr) or getattr(self, attr) is None:
1763             setattr(self, attr, value)
1764         return getattr(self, attr)
1765
1766     def defaultDestination(self, args):
1767         ## TODO: use common prefix of args?
1768         depotPath = args[0]
1769         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1770         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1771         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1772         depotDir = re.sub(r"/$", "", depotDir)
1773         return os.path.split(depotDir)[1]
1774
1775     def run(self, args):
1776         if len(args) < 1:
1777             return False
1778
1779         if self.keepRepoPath and not self.cloneDestination:
1780             sys.stderr.write("Must specify destination for --keep-path\n")
1781             sys.exit(1)
1782
1783         depotPaths = args
1784
1785         if not self.cloneDestination and len(depotPaths) > 1:
1786             self.cloneDestination = depotPaths[-1]
1787             depotPaths = depotPaths[:-1]
1788
1789         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1790         for p in depotPaths:
1791             if not p.startswith("//"):
1792                 return False
1793
1794         if not self.cloneDestination:
1795             self.cloneDestination = self.defaultDestination(args)
1796
1797         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1798         if not os.path.exists(self.cloneDestination):
1799             os.makedirs(self.cloneDestination)
1800         chdir(self.cloneDestination)
1801         system("git init")
1802         self.gitdir = os.getcwd() + "/.git"
1803         if not P4Sync.run(self, depotPaths):
1804             return False
1805         if self.branch != "master":
1806             if self.importIntoRemotes:
1807                 masterbranch = "refs/remotes/p4/master"
1808             else:
1809                 masterbranch = "refs/heads/p4/master"
1810             if gitBranchExists(masterbranch):
1811                 system("git branch master %s" % masterbranch)
1812                 system("git checkout -f")
1813             else:
1814                 print "Could not detect main branch. No checkout/master branch created."
1815
1816         return True
1817
1818 class P4Branches(Command):
1819     def __init__(self):
1820         Command.__init__(self)
1821         self.options = [ ]
1822         self.description = ("Shows the git branches that hold imports and their "
1823                             + "corresponding perforce depot paths")
1824         self.verbose = False
1825
1826     def run(self, args):
1827         if originP4BranchesExist():
1828             createOrUpdateBranchesFromOrigin()
1829
1830         cmdline = "git rev-parse --symbolic "
1831         cmdline += " --remotes"
1832
1833         for line in read_pipe_lines(cmdline):
1834             line = line.strip()
1835
1836             if not line.startswith('p4/') or line == "p4/HEAD":
1837                 continue
1838             branch = line
1839
1840             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1841             settings = extractSettingsGitLog(log)
1842
1843             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1844         return True
1845
1846 class HelpFormatter(optparse.IndentedHelpFormatter):
1847     def __init__(self):
1848         optparse.IndentedHelpFormatter.__init__(self)
1849
1850     def format_description(self, description):
1851         if description:
1852             return description + "\n"
1853         else:
1854             return ""
1855
1856 def printUsage(commands):
1857     print "usage: %s <command> [options]" % sys.argv[0]
1858     print ""
1859     print "valid commands: %s" % ", ".join(commands)
1860     print ""
1861     print "Try %s <command> --help for command specific help." % sys.argv[0]
1862     print ""
1863
1864 commands = {
1865     "debug" : P4Debug,
1866     "submit" : P4Submit,
1867     "commit" : P4Submit,
1868     "sync" : P4Sync,
1869     "rebase" : P4Rebase,
1870     "clone" : P4Clone,
1871     "rollback" : P4RollBack,
1872     "branches" : P4Branches
1873 }
1874
1875
1876 def main():
1877     if len(sys.argv[1:]) == 0:
1878         printUsage(commands.keys())
1879         sys.exit(2)
1880
1881     cmd = ""
1882     cmdName = sys.argv[1]
1883     try:
1884         klass = commands[cmdName]
1885         cmd = klass()
1886     except KeyError:
1887         print "unknown command %s" % cmdName
1888         print ""
1889         printUsage(commands.keys())
1890         sys.exit(2)
1891
1892     options = cmd.options
1893     cmd.gitdir = os.environ.get("GIT_DIR", None)
1894
1895     args = sys.argv[2:]
1896
1897     if len(options) > 0:
1898         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1899
1900         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1901                                        options,
1902                                        description = cmd.description,
1903                                        formatter = HelpFormatter())
1904
1905         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1906     global verbose
1907     verbose = cmd.verbose
1908     if cmd.needsGit:
1909         if cmd.gitdir == None:
1910             cmd.gitdir = os.path.abspath(".git")
1911             if not isValidGitDir(cmd.gitdir):
1912                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1913                 if os.path.exists(cmd.gitdir):
1914                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1915                     if len(cdup) > 0:
1916                         chdir(cdup);
1917
1918         if not isValidGitDir(cmd.gitdir):
1919             if isValidGitDir(cmd.gitdir + "/.git"):
1920                 cmd.gitdir += "/.git"
1921             else:
1922                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1923
1924         os.environ["GIT_DIR"] = cmd.gitdir
1925
1926     if not cmd.run(args):
1927         parser.print_help()
1928
1929
1930 if __name__ == '__main__':
1931     main()