contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = ["p4"]
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += ["-u",user]
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += ["-P", password]
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += ["-p", port]
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += ["-h", host]
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += ["-c", client]
  46
  47
  48     if isinstance(cmd,basestring):
  49         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50     else:
  51         real_cmd += cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56     # not using the shell, we have to set it ourselves.  This path could
  57     # be relative, so go there first, then figure out where we ended up.
  58     os.chdir(dir)
  59     os.environ['PWD'] = os.getcwd()
  60
  61 def die(msg):
  62     if verbose:
  63         raise Exception(msg)
  64     else:
  65         sys.stderr.write(msg + "\n")
  66         sys.exit(1)
  67
  68 def write_pipe(c, stdin):
  69     if verbose:
  70         sys.stderr.write('Writing pipe: %s\n' % str(c))
  71
  72     expand = isinstance(c,basestring)
  73     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  74     pipe = p.stdin
  75     val = pipe.write(stdin)
  76     pipe.close()
  77     if p.wait():
  78         die('Command failed: %s' % str(c))
  79
  80     return val
  81
  82 def p4_write_pipe(c, stdin):
  83     real_cmd = p4_build_cmd(c)
  84     return write_pipe(real_cmd, stdin)
  85
  86 def read_pipe(c, ignore_error=False):
  87     if verbose:
  88         sys.stderr.write('Reading pipe: %s\n' % str(c))
  89
  90     expand = isinstance(c,basestring)
  91     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  92     pipe = p.stdout
  93     val = pipe.read()
  94     if p.wait() and not ignore_error:
  95         die('Command failed: %s' % str(c))
  96
  97     return val
  98
  99 def p4_read_pipe(c, ignore_error=False):
 100     real_cmd = p4_build_cmd(c)
 101     return read_pipe(real_cmd, ignore_error)
 102
 103 def read_pipe_lines(c):
 104     if verbose:
 105         sys.stderr.write('Reading pipe: %s\n' % str(c))
 106
 107     expand = isinstance(c, basestring)
 108     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 109     pipe = p.stdout
 110     val = pipe.readlines()
 111     if pipe.close() or p.wait():
 112         die('Command failed: %s' % str(c))
 113
 114     return val
 115
 116 def p4_read_pipe_lines(c):
 117     """Specifically invoke p4 on the command supplied. """
 118     real_cmd = p4_build_cmd(c)
 119     return read_pipe_lines(real_cmd)
 120
 121 def system(cmd):
 122     expand = isinstance(cmd,basestring)
 123     if verbose:
 124         sys.stderr.write("executing %s\n" % str(cmd))
 125     subprocess.check_call(cmd, shell=expand)
 126
 127 def p4_system(cmd):
 128     """Specifically invoke p4 as the system command. """
 129     real_cmd = p4_build_cmd(cmd)
 130     expand = isinstance(real_cmd, basestring)
 131     subprocess.check_call(real_cmd, shell=expand)
 132
 133 def p4_integrate(src, dest):
 134     p4_system(["integrate", "-Dt", src, dest])
 135
 136 def p4_sync(path):
 137     p4_system(["sync", path])
 138
 139 def p4_add(f):
 140     p4_system(["add", f])
 141
 142 def p4_delete(f):
 143     p4_system(["delete", f])
 144
 145 def p4_edit(f):
 146     p4_system(["edit", f])
 147
 148 def p4_revert(f):
 149     p4_system(["revert", f])
 150
 151 def p4_reopen(type, file):
 152     p4_system(["reopen", "-t", type, file])
 153
 154 #
 155 # Canonicalize the p4 type and return a tuple of the
 156 # base type, plus any modifiers.  See "p4 help filetypes"
 157 # for a list and explanation.
 158 #
 159 def split_p4_type(p4type):
 160
 161     p4_filetypes_historical = {
 162         "ctempobj": "binary+Sw",
 163         "ctext": "text+C",
 164         "cxtext": "text+Cx",
 165         "ktext": "text+k",
 166         "kxtext": "text+kx",
 167         "ltext": "text+F",
 168         "tempobj": "binary+FSw",
 169         "ubinary": "binary+F",
 170         "uresource": "resource+F",
 171         "uxbinary": "binary+Fx",
 172         "xbinary": "binary+x",
 173         "xltext": "text+Fx",
 174         "xtempobj": "binary+Swx",
 175         "xtext": "text+x",
 176         "xunicode": "unicode+x",
 177         "xutf16": "utf16+x",
 178     }
 179     if p4type in p4_filetypes_historical:
 180         p4type = p4_filetypes_historical[p4type]
 181     mods = ""
 182     s = p4type.split("+")
 183     base = s[0]
 184     mods = ""
 185     if len(s) > 1:
 186         mods = s[1]
 187     return (base, mods)
 188
 189
 190 def setP4ExecBit(file, mode):
 191     # Reopens an already open file and changes the execute bit to match
 192     # the execute bit setting in the passed in mode.
 193
 194     p4Type = "+x"
 195
 196     if not isModeExec(mode):
 197         p4Type = getP4OpenedType(file)
 198         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 199         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 200         if p4Type[-1] == "+":
 201             p4Type = p4Type[0:-1]
 202
 203     p4_reopen(p4Type, file)
 204
 205 def getP4OpenedType(file):
 206     # Returns the perforce file type for the given file.
 207
 208     result = p4_read_pipe(["opened", file])
 209     match = re.match(".*\((.+)\)\r?$", result)
 210     if match:
 211         return match.group(1)
 212     else:
 213         die("Could not determine file type for %s (result: '%s')" % (file, result))
 214
 215 def diffTreePattern():
 216     # This is a simple generator for the diff tree regex pattern. This could be
 217     # a class variable if this and parseDiffTreeEntry were a part of a class.
 218     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 219     while True:
 220         yield pattern
 221
 222 def parseDiffTreeEntry(entry):
 223     """Parses a single diff tree entry into its component elements.
 224
 225     See git-diff-tree(1) manpage for details about the format of the diff
 226     output. This method returns a dictionary with the following elements:
 227
 228     src_mode - The mode of the source file
 229     dst_mode - The mode of the destination file
 230     src_sha1 - The sha1 for the source file
 231     dst_sha1 - The sha1 fr the destination file
 232     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 233     status_score - The score for the status (applicable for 'C' and 'R'
 234                    statuses). This is None if there is no score.
 235     src - The path for the source file.
 236     dst - The path for the destination file. This is only present for
 237           copy or renames. If it is not present, this is None.
 238
 239     If the pattern is not matched, None is returned."""
 240
 241     match = diffTreePattern().next().match(entry)
 242     if match:
 243         return {
 244             'src_mode': match.group(1),
 245             'dst_mode': match.group(2),
 246             'src_sha1': match.group(3),
 247             'dst_sha1': match.group(4),
 248             'status': match.group(5),
 249             'status_score': match.group(6),
 250             'src': match.group(7),
 251             'dst': match.group(10)
 252         }
 253     return None
 254
 255 def isModeExec(mode):
 256     # Returns True if the given git mode represents an executable file,
 257     # otherwise False.
 258     return mode[-3:] == "755"
 259
 260 def isModeExecChanged(src_mode, dst_mode):
 261     return isModeExec(src_mode) != isModeExec(dst_mode)
 262
 263 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 264
 265     if isinstance(cmd,basestring):
 266         cmd = "-G " + cmd
 267         expand = True
 268     else:
 269         cmd = ["-G"] + cmd
 270         expand = False
 271
 272     cmd = p4_build_cmd(cmd)
 273     if verbose:
 274         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 275
 276     # Use a temporary file to avoid deadlocks without
 277     # subprocess.communicate(), which would put another copy
 278     # of stdout into memory.
 279     stdin_file = None
 280     if stdin is not None:
 281         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 282         if isinstance(stdin,basestring):
 283             stdin_file.write(stdin)
 284         else:
 285             for i in stdin:
 286                 stdin_file.write(i + '\n')
 287         stdin_file.flush()
 288         stdin_file.seek(0)
 289
 290     p4 = subprocess.Popen(cmd,
 291                           shell=expand,
 292                           stdin=stdin_file,
 293                           stdout=subprocess.PIPE)
 294
 295     result = []
 296     try:
 297         while True:
 298             entry = marshal.load(p4.stdout)
 299             if cb is not None:
 300                 cb(entry)
 301             else:
 302                 result.append(entry)
 303     except EOFError:
 304         pass
 305     exitCode = p4.wait()
 306     if exitCode != 0:
 307         entry = {}
 308         entry["p4ExitCode"] = exitCode
 309         result.append(entry)
 310
 311     return result
 312
 313 def p4Cmd(cmd):
 314     list = p4CmdList(cmd)
 315     result = {}
 316     for entry in list:
 317         result.update(entry)
 318     return result;
 319
 320 def p4Where(depotPath):
 321     if not depotPath.endswith("/"):
 322         depotPath += "/"
 323     depotPath = depotPath + "..."
 324     outputList = p4CmdList(["where", depotPath])
 325     output = None
 326     for entry in outputList:
 327         if "depotFile" in entry:
 328             if entry["depotFile"] == depotPath:
 329                 output = entry
 330                 break
 331         elif "data" in entry:
 332             data = entry.get("data")
 333             space = data.find(" ")
 334             if data[:space] == depotPath:
 335                 output = entry
 336                 break
 337     if output == None:
 338         return ""
 339     if output["code"] == "error":
 340         return ""
 341     clientPath = ""
 342     if "path" in output:
 343         clientPath = output.get("path")
 344     elif "data" in output:
 345         data = output.get("data")
 346         lastSpace = data.rfind(" ")
 347         clientPath = data[lastSpace + 1:]
 348
 349     if clientPath.endswith("..."):
 350         clientPath = clientPath[:-3]
 351     return clientPath
 352
 353 def currentGitBranch():
 354     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 355
 356 def isValidGitDir(path):
 357     if (os.path.exists(path + "/HEAD")
 358         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 359         return True;
 360     return False
 361
 362 def parseRevision(ref):
 363     return read_pipe("git rev-parse %s" % ref).strip()
 364
 365 def branchExists(ref):
 366     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 367                      ignore_error=True)
 368     return len(rev) > 0
 369
 370 def extractLogMessageFromGitCommit(commit):
 371     logMessage = ""
 372
 373     ## fixme: title is first line of commit, not 1st paragraph.
 374     foundTitle = False
 375     for log in read_pipe_lines("git cat-file commit %s" % commit):
 376        if not foundTitle:
 377            if len(log) == 1:
 378                foundTitle = True
 379            continue
 380
 381        logMessage += log
 382     return logMessage
 383
 384 def extractSettingsGitLog(log):
 385     values = {}
 386     for line in log.split("\n"):
 387         line = line.strip()
 388         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 389         if not m:
 390             continue
 391
 392         assignments = m.group(1).split (':')
 393         for a in assignments:
 394             vals = a.split ('=')
 395             key = vals[0].strip()
 396             val = ('='.join (vals[1:])).strip()
 397             if val.endswith ('\"') and val.startswith('"'):
 398                 val = val[1:-1]
 399
 400             values[key] = val
 401
 402     paths = values.get("depot-paths")
 403     if not paths:
 404         paths = values.get("depot-path")
 405     if paths:
 406         values['depot-paths'] = paths.split(',')
 407     return values
 408
 409 def gitBranchExists(branch):
 410     proc = subprocess.Popen(["git", "rev-parse", branch],
 411                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 412     return proc.wait() == 0;
 413
 414 _gitConfig = {}
 415 def gitConfig(key, args = None): # set args to "--bool", for instance
 416     if not _gitConfig.has_key(key):
 417         argsFilter = ""
 418         if args != None:
 419             argsFilter = "%s " % args
 420         cmd = "git config %s%s" % (argsFilter, key)
 421         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 422     return _gitConfig[key]
 423
 424 def gitConfigList(key):
 425     if not _gitConfig.has_key(key):
 426         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 427     return _gitConfig[key]
 428
 429 def p4BranchesInGit(branchesAreInRemotes = True):
 430     branches = {}
 431
 432     cmdline = "git rev-parse --symbolic "
 433     if branchesAreInRemotes:
 434         cmdline += " --remotes"
 435     else:
 436         cmdline += " --branches"
 437
 438     for line in read_pipe_lines(cmdline):
 439         line = line.strip()
 440
 441         ## only import to p4/
 442         if not line.startswith('p4/') or line == "p4/HEAD":
 443             continue
 444         branch = line
 445
 446         # strip off p4
 447         branch = re.sub ("^p4/", "", line)
 448
 449         branches[branch] = parseRevision(line)
 450     return branches
 451
 452 def findUpstreamBranchPoint(head = "HEAD"):
 453     branches = p4BranchesInGit()
 454     # map from depot-path to branch name
 455     branchByDepotPath = {}
 456     for branch in branches.keys():
 457         tip = branches[branch]
 458         log = extractLogMessageFromGitCommit(tip)
 459         settings = extractSettingsGitLog(log)
 460         if settings.has_key("depot-paths"):
 461             paths = ",".join(settings["depot-paths"])
 462             branchByDepotPath[paths] = "remotes/p4/" + branch
 463
 464     settings = None
 465     parent = 0
 466     while parent < 65535:
 467         commit = head + "~%s" % parent
 468         log = extractLogMessageFromGitCommit(commit)
 469         settings = extractSettingsGitLog(log)
 470         if settings.has_key("depot-paths"):
 471             paths = ",".join(settings["depot-paths"])
 472             if branchByDepotPath.has_key(paths):
 473                 return [branchByDepotPath[paths], settings]
 474
 475         parent = parent + 1
 476
 477     return ["", settings]
 478
 479 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 480     if not silent:
 481         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 482                % localRefPrefix)
 483
 484     originPrefix = "origin/p4/"
 485
 486     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 487         line = line.strip()
 488         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 489             continue
 490
 491         headName = line[len(originPrefix):]
 492         remoteHead = localRefPrefix + headName
 493         originHead = line
 494
 495         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 496         if (not original.has_key('depot-paths')
 497             or not original.has_key('change')):
 498             continue
 499
 500         update = False
 501         if not gitBranchExists(remoteHead):
 502             if verbose:
 503                 print "creating %s" % remoteHead
 504             update = True
 505         else:
 506             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 507             if settings.has_key('change') > 0:
 508                 if settings['depot-paths'] == original['depot-paths']:
 509                     originP4Change = int(original['change'])
 510                     p4Change = int(settings['change'])
 511                     if originP4Change > p4Change:
 512                         print ("%s (%s) is newer than %s (%s). "
 513                                "Updating p4 branch from origin."
 514                                % (originHead, originP4Change,
 515                                   remoteHead, p4Change))
 516                         update = True
 517                 else:
 518                     print ("Ignoring: %s was imported from %s while "
 519                            "%s was imported from %s"
 520                            % (originHead, ','.join(original['depot-paths']),
 521                               remoteHead, ','.join(settings['depot-paths'])))
 522
 523         if update:
 524             system("git update-ref %s %s" % (remoteHead, originHead))
 525
 526 def originP4BranchesExist():
 527         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 528
 529 def p4ChangesForPaths(depotPaths, changeRange):
 530     assert depotPaths
 531     cmd = ['changes']
 532     for p in depotPaths:
 533         cmd += ["%s...%s" % (p, changeRange)]
 534     output = p4_read_pipe_lines(cmd)
 535
 536     changes = {}
 537     for line in output:
 538         changeNum = int(line.split(" ")[1])
 539         changes[changeNum] = True
 540
 541     changelist = changes.keys()
 542     changelist.sort()
 543     return changelist
 544
 545 def p4PathStartsWith(path, prefix):
 546     # This method tries to remedy a potential mixed-case issue:
 547     #
 548     # If UserA adds  //depot/DirA/file1
 549     # and UserB adds //depot/dira/file2
 550     #
 551     # we may or may not have a problem. If you have core.ignorecase=true,
 552     # we treat DirA and dira as the same directory
 553     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 554     if ignorecase:
 555         return path.lower().startswith(prefix.lower())
 556     return path.startswith(prefix)
 557
 558 def getClientSpec():
 559     """Look at the p4 client spec, create a View() object that contains
 560        all the mappings, and return it."""
 561
 562     specList = p4CmdList("client -o")
 563     if len(specList) != 1:
 564         die('Output from "client -o" is %d lines, expecting 1' %
 565             len(specList))
 566
 567     # dictionary of all client parameters
 568     entry = specList[0]
 569
 570     # just the keys that start with "View"
 571     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 572
 573     # hold this new View
 574     view = View()
 575
 576     # append the lines, in order, to the view
 577     for view_num in range(len(view_keys)):
 578         k = "View%d" % view_num
 579         if k not in view_keys:
 580             die("Expected view key %s missing" % k)
 581         view.append(entry[k])
 582
 583     return view
 584
 585 def getClientRoot():
 586     """Grab the client directory."""
 587
 588     output = p4CmdList("client -o")
 589     if len(output) != 1:
 590         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 591
 592     entry = output[0]
 593     if "Root" not in entry:
 594         die('Client has no "Root"')
 595
 596     return entry["Root"]
 597
 598 class Command:
 599     def __init__(self):
 600         self.usage = "usage: %prog [options]"
 601         self.needsGit = True
 602
 603 class P4UserMap:
 604     def __init__(self):
 605         self.userMapFromPerforceServer = False
 606
 607     def getUserCacheFilename(self):
 608         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 609         return home + "/.gitp4-usercache.txt"
 610
 611     def getUserMapFromPerforceServer(self):
 612         if self.userMapFromPerforceServer:
 613             return
 614         self.users = {}
 615         self.emails = {}
 616
 617         for output in p4CmdList("users"):
 618             if not output.has_key("User"):
 619                 continue
 620             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 621             self.emails[output["Email"]] = output["User"]
 622
 623
 624         s = ''
 625         for (key, val) in self.users.items():
 626             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 627
 628         open(self.getUserCacheFilename(), "wb").write(s)
 629         self.userMapFromPerforceServer = True
 630
 631     def loadUserMapFromCache(self):
 632         self.users = {}
 633         self.userMapFromPerforceServer = False
 634         try:
 635             cache = open(self.getUserCacheFilename(), "rb")
 636             lines = cache.readlines()
 637             cache.close()
 638             for line in lines:
 639                 entry = line.strip().split("\t")
 640                 self.users[entry[0]] = entry[1]
 641         except IOError:
 642             self.getUserMapFromPerforceServer()
 643
 644 class P4Debug(Command):
 645     def __init__(self):
 646         Command.__init__(self)
 647         self.options = [
 648             optparse.make_option("--verbose", dest="verbose", action="store_true",
 649                                  default=False),
 650             ]
 651         self.description = "A tool to debug the output of p4 -G."
 652         self.needsGit = False
 653         self.verbose = False
 654
 655     def run(self, args):
 656         j = 0
 657         for output in p4CmdList(args):
 658             print 'Element: %d' % j
 659             j += 1
 660             print output
 661         return True
 662
 663 class P4RollBack(Command):
 664     def __init__(self):
 665         Command.__init__(self)
 666         self.options = [
 667             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 668             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 669         ]
 670         self.description = "A tool to debug the multi-branch import. Don't use :)"
 671         self.verbose = False
 672         self.rollbackLocalBranches = False
 673
 674     def run(self, args):
 675         if len(args) != 1:
 676             return False
 677         maxChange = int(args[0])
 678
 679         if "p4ExitCode" in p4Cmd("changes -m 1"):
 680             die("Problems executing p4");
 681
 682         if self.rollbackLocalBranches:
 683             refPrefix = "refs/heads/"
 684             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 685         else:
 686             refPrefix = "refs/remotes/"
 687             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 688
 689         for line in lines:
 690             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 691                 line = line.strip()
 692                 ref = refPrefix + line
 693                 log = extractLogMessageFromGitCommit(ref)
 694                 settings = extractSettingsGitLog(log)
 695
 696                 depotPaths = settings['depot-paths']
 697                 change = settings['change']
 698
 699                 changed = False
 700
 701                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 702                                                            for p in depotPaths]))) == 0:
 703                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 704                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 705                     continue
 706
 707                 while change and int(change) > maxChange:
 708                     changed = True
 709                     if self.verbose:
 710                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 711                     system("git update-ref %s \"%s^\"" % (ref, ref))
 712                     log = extractLogMessageFromGitCommit(ref)
 713                     settings =  extractSettingsGitLog(log)
 714
 715
 716                     depotPaths = settings['depot-paths']
 717                     change = settings['change']
 718
 719                 if changed:
 720                     print "%s rewound to %s" % (ref, change)
 721
 722         return True
 723
 724 class P4Submit(Command, P4UserMap):
 725     def __init__(self):
 726         Command.__init__(self)
 727         P4UserMap.__init__(self)
 728         self.options = [
 729                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 730                 optparse.make_option("--origin", dest="origin"),
 731                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 732                 # preserve the user, requires relevant p4 permissions
 733                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 734         ]
 735         self.description = "Submit changes from git to the perforce depot."
 736         self.usage += " [name of git branch to submit into perforce depot]"
 737         self.interactive = True
 738         self.origin = ""
 739         self.detectRenames = False
 740         self.verbose = False
 741         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 742         self.isWindows = (platform.system() == "Windows")
 743         self.myP4UserId = None
 744
 745     def check(self):
 746         if len(p4CmdList("opened ...")) > 0:
 747             die("You have files opened with perforce! Close them before starting the sync.")
 748
 749     # replaces everything between 'Description:' and the next P4 submit template field with the
 750     # commit message
 751     def prepareLogMessage(self, template, message):
 752         result = ""
 753
 754         inDescriptionSection = False
 755
 756         for line in template.split("\n"):
 757             if line.startswith("#"):
 758                 result += line + "\n"
 759                 continue
 760
 761             if inDescriptionSection:
 762                 if line.startswith("Files:") or line.startswith("Jobs:"):
 763                     inDescriptionSection = False
 764                 else:
 765                     continue
 766             else:
 767                 if line.startswith("Description:"):
 768                     inDescriptionSection = True
 769                     line += "\n"
 770                     for messageLine in message.split("\n"):
 771                         line += "\t" + messageLine + "\n"
 772
 773             result += line + "\n"
 774
 775         return result
 776
 777     def p4UserForCommit(self,id):
 778         # Return the tuple (perforce user,git email) for a given git commit id
 779         self.getUserMapFromPerforceServer()
 780         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 781         gitEmail = gitEmail.strip()
 782         if not self.emails.has_key(gitEmail):
 783             return (None,gitEmail)
 784         else:
 785             return (self.emails[gitEmail],gitEmail)
 786
 787     def checkValidP4Users(self,commits):
 788         # check if any git authors cannot be mapped to p4 users
 789         for id in commits:
 790             (user,email) = self.p4UserForCommit(id)
 791             if not user:
 792                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 793                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 794                     print "%s" % msg
 795                 else:
 796                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 797
 798     def lastP4Changelist(self):
 799         # Get back the last changelist number submitted in this client spec. This
 800         # then gets used to patch up the username in the change. If the same
 801         # client spec is being used by multiple processes then this might go
 802         # wrong.
 803         results = p4CmdList("client -o")        # find the current client
 804         client = None
 805         for r in results:
 806             if r.has_key('Client'):
 807                 client = r['Client']
 808                 break
 809         if not client:
 810             die("could not get client spec")
 811         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 812         for r in results:
 813             if r.has_key('change'):
 814                 return r['change']
 815         die("Could not get changelist number for last submit - cannot patch up user details")
 816
 817     def modifyChangelistUser(self, changelist, newUser):
 818         # fixup the user field of a changelist after it has been submitted.
 819         changes = p4CmdList("change -o %s" % changelist)
 820         if len(changes) != 1:
 821             die("Bad output from p4 change modifying %s to user %s" %
 822                 (changelist, newUser))
 823
 824         c = changes[0]
 825         if c['User'] == newUser: return   # nothing to do
 826         c['User'] = newUser
 827         input = marshal.dumps(c)
 828
 829         result = p4CmdList("change -f -i", stdin=input)
 830         for r in result:
 831             if r.has_key('code'):
 832                 if r['code'] == 'error':
 833                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 834             if r.has_key('data'):
 835                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 836                 return
 837         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 838
 839     def canChangeChangelists(self):
 840         # check to see if we have p4 admin or super-user permissions, either of
 841         # which are required to modify changelists.
 842         results = p4CmdList("protects %s" % self.depotPath)
 843         for r in results:
 844             if r.has_key('perm'):
 845                 if r['perm'] == 'admin':
 846                     return 1
 847                 if r['perm'] == 'super':
 848                     return 1
 849         return 0
 850
 851     def p4UserId(self):
 852         if self.myP4UserId:
 853             return self.myP4UserId
 854
 855         results = p4CmdList("user -o")
 856         for r in results:
 857             if r.has_key('User'):
 858                 self.myP4UserId = r['User']
 859                 return r['User']
 860         die("Could not find your p4 user id")
 861
 862     def p4UserIsMe(self, p4User):
 863         # return True if the given p4 user is actually me
 864         me = self.p4UserId()
 865         if not p4User or p4User != me:
 866             return False
 867         else:
 868             return True
 869
 870     def prepareSubmitTemplate(self):
 871         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 872         template = ""
 873         inFilesSection = False
 874         for line in p4_read_pipe_lines(['change', '-o']):
 875             if line.endswith("\r\n"):
 876                 line = line[:-2] + "\n"
 877             if inFilesSection:
 878                 if line.startswith("\t"):
 879                     # path starts and ends with a tab
 880                     path = line[1:]
 881                     lastTab = path.rfind("\t")
 882                     if lastTab != -1:
 883                         path = path[:lastTab]
 884                         if not p4PathStartsWith(path, self.depotPath):
 885                             continue
 886                 else:
 887                     inFilesSection = False
 888             else:
 889                 if line.startswith("Files:"):
 890                     inFilesSection = True
 891
 892             template += line
 893
 894         return template
 895
 896     def edit_template(self, template_file):
 897         """Invoke the editor to let the user change the submission
 898            message.  Return true if okay to continue with the submit."""
 899
 900         # if configured to skip the editing part, just submit
 901         if gitConfig("git-p4.skipSubmitEdit") == "true":
 902             return True
 903
 904         # look at the modification time, to check later if the user saved
 905         # the file
 906         mtime = os.stat(template_file).st_mtime
 907
 908         # invoke the editor
 909         if os.environ.has_key("P4EDITOR"):
 910             editor = os.environ.get("P4EDITOR")
 911         else:
 912             editor = read_pipe("git var GIT_EDITOR").strip()
 913         system(editor + " " + template_file)
 914
 915         # If the file was not saved, prompt to see if this patch should
 916         # be skipped.  But skip this verification step if configured so.
 917         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 918             return True
 919
 920         # modification time updated means user saved the file
 921         if os.stat(template_file).st_mtime > mtime:
 922             return True
 923
 924         while True:
 925             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 926             if response == 'y':
 927                 return True
 928             if response == 'n':
 929                 return False
 930
 931     def applyCommit(self, id):
 932         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 933
 934         (p4User, gitEmail) = self.p4UserForCommit(id)
 935
 936         if not self.detectRenames:
 937             # If not explicitly set check the config variable
 938             self.detectRenames = gitConfig("git-p4.detectRenames")
 939
 940         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 941             diffOpts = ""
 942         elif self.detectRenames.lower() == "true":
 943             diffOpts = "-M"
 944         else:
 945             diffOpts = "-M%s" % self.detectRenames
 946
 947         detectCopies = gitConfig("git-p4.detectCopies")
 948         if detectCopies.lower() == "true":
 949             diffOpts += " -C"
 950         elif detectCopies != "" and detectCopies.lower() != "false":
 951             diffOpts += " -C%s" % detectCopies
 952
 953         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 954             diffOpts += " --find-copies-harder"
 955
 956         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 957         filesToAdd = set()
 958         filesToDelete = set()
 959         editedFiles = set()
 960         filesToChangeExecBit = {}
 961         for line in diff:
 962             diff = parseDiffTreeEntry(line)
 963             modifier = diff['status']
 964             path = diff['src']
 965             if modifier == "M":
 966                 p4_edit(path)
 967                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 968                     filesToChangeExecBit[path] = diff['dst_mode']
 969                 editedFiles.add(path)
 970             elif modifier == "A":
 971                 filesToAdd.add(path)
 972                 filesToChangeExecBit[path] = diff['dst_mode']
 973                 if path in filesToDelete:
 974                     filesToDelete.remove(path)
 975             elif modifier == "D":
 976                 filesToDelete.add(path)
 977                 if path in filesToAdd:
 978                     filesToAdd.remove(path)
 979             elif modifier == "C":
 980                 src, dest = diff['src'], diff['dst']
 981                 p4_integrate(src, dest)
 982                 if diff['src_sha1'] != diff['dst_sha1']:
 983                     p4_edit(dest)
 984                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 985                     p4_edit(dest)
 986                     filesToChangeExecBit[dest] = diff['dst_mode']
 987                 os.unlink(dest)
 988                 editedFiles.add(dest)
 989             elif modifier == "R":
 990                 src, dest = diff['src'], diff['dst']
 991                 p4_integrate(src, dest)
 992                 if diff['src_sha1'] != diff['dst_sha1']:
 993                     p4_edit(dest)
 994                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 995                     p4_edit(dest)
 996                     filesToChangeExecBit[dest] = diff['dst_mode']
 997                 os.unlink(dest)
 998                 editedFiles.add(dest)
 999                 filesToDelete.add(src)
1000             else:
1001                 die("unknown modifier %s for %s" % (modifier, path))
1002
1003         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1004         patchcmd = diffcmd + " | git apply "
1005         tryPatchCmd = patchcmd + "--check -"
1006         applyPatchCmd = patchcmd + "--check --apply -"
1007
1008         if os.system(tryPatchCmd) != 0:
1009             print "Unfortunately applying the change failed!"
1010             print "What do you want to do?"
1011             response = "x"
1012             while response != "s" and response != "a" and response != "w":
1013                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
1014                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1015             if response == "s":
1016                 print "Skipping! Good luck with the next patches..."
1017                 for f in editedFiles:
1018                     p4_revert(f)
1019                 for f in filesToAdd:
1020                     os.remove(f)
1021                 return
1022             elif response == "a":
1023                 os.system(applyPatchCmd)
1024                 if len(filesToAdd) > 0:
1025                     print "You may also want to call p4 add on the following files:"
1026                     print " ".join(filesToAdd)
1027                 if len(filesToDelete):
1028                     print "The following files should be scheduled for deletion with p4 delete:"
1029                     print " ".join(filesToDelete)
1030                 die("Please resolve and submit the conflict manually and "
1031                     + "continue afterwards with git-p4 submit --continue")
1032             elif response == "w":
1033                 system(diffcmd + " > patch.txt")
1034                 print "Patch saved to patch.txt in %s !" % self.clientPath
1035                 die("Please resolve and submit the conflict manually and "
1036                     "continue afterwards with git-p4 submit --continue")
1037
1038         system(applyPatchCmd)
1039
1040         for f in filesToAdd:
1041             p4_add(f)
1042         for f in filesToDelete:
1043             p4_revert(f)
1044             p4_delete(f)
1045
1046         # Set/clear executable bits
1047         for f in filesToChangeExecBit.keys():
1048             mode = filesToChangeExecBit[f]
1049             setP4ExecBit(f, mode)
1050
1051         logMessage = extractLogMessageFromGitCommit(id)
1052         logMessage = logMessage.strip()
1053
1054         template = self.prepareSubmitTemplate()
1055
1056         if self.interactive:
1057             submitTemplate = self.prepareLogMessage(template, logMessage)
1058
1059             if self.preserveUser:
1060                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1061
1062             if os.environ.has_key("P4DIFF"):
1063                 del(os.environ["P4DIFF"])
1064             diff = ""
1065             for editedFile in editedFiles:
1066                 diff += p4_read_pipe(['diff', '-du', editedFile])
1067
1068             newdiff = ""
1069             for newFile in filesToAdd:
1070                 newdiff += "==== new file ====\n"
1071                 newdiff += "--- /dev/null\n"
1072                 newdiff += "+++ %s\n" % newFile
1073                 f = open(newFile, "r")
1074                 for line in f.readlines():
1075                     newdiff += "+" + line
1076                 f.close()
1077
1078             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1079                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1080                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1081                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1082
1083             separatorLine = "######## everything below this line is just the diff #######\n"
1084
1085             (handle, fileName) = tempfile.mkstemp()
1086             tmpFile = os.fdopen(handle, "w+")
1087             if self.isWindows:
1088                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1089                 separatorLine = separatorLine.replace("\n", "\r\n")
1090                 newdiff = newdiff.replace("\n", "\r\n")
1091             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1092             tmpFile.close()
1093
1094             if self.edit_template(fileName):
1095                 # read the edited message and submit
1096                 tmpFile = open(fileName, "rb")
1097                 message = tmpFile.read()
1098                 tmpFile.close()
1099                 submitTemplate = message[:message.index(separatorLine)]
1100                 if self.isWindows:
1101                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1102                 p4_write_pipe(['submit', '-i'], submitTemplate)
1103
1104                 if self.preserveUser:
1105                     if p4User:
1106                         # Get last changelist number. Cannot easily get it from
1107                         # the submit command output as the output is
1108                         # unmarshalled.
1109                         changelist = self.lastP4Changelist()
1110                         self.modifyChangelistUser(changelist, p4User)
1111             else:
1112                 # skip this patch
1113                 print "Submission cancelled, undoing p4 changes."
1114                 for f in editedFiles:
1115                     p4_revert(f)
1116                 for f in filesToAdd:
1117                     p4_revert(f)
1118                     os.remove(f)
1119
1120             os.remove(fileName)
1121         else:
1122             fileName = "submit.txt"
1123             file = open(fileName, "w+")
1124             file.write(self.prepareLogMessage(template, logMessage))
1125             file.close()
1126             print ("Perforce submit template written as %s. "
1127                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1128                    % (fileName, fileName))
1129
1130     def run(self, args):
1131         if len(args) == 0:
1132             self.master = currentGitBranch()
1133             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1134                 die("Detecting current git branch failed!")
1135         elif len(args) == 1:
1136             self.master = args[0]
1137             if not branchExists(self.master):
1138                 die("Branch %s does not exist" % self.master)
1139         else:
1140             return False
1141
1142         allowSubmit = gitConfig("git-p4.allowSubmit")
1143         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1144             die("%s is not in git-p4.allowSubmit" % self.master)
1145
1146         [upstream, settings] = findUpstreamBranchPoint()
1147         self.depotPath = settings['depot-paths'][0]
1148         if len(self.origin) == 0:
1149             self.origin = upstream
1150
1151         if self.preserveUser:
1152             if not self.canChangeChangelists():
1153                 die("Cannot preserve user names without p4 super-user or admin permissions")
1154
1155         if self.verbose:
1156             print "Origin branch is " + self.origin
1157
1158         if len(self.depotPath) == 0:
1159             print "Internal error: cannot locate perforce depot path from existing branches"
1160             sys.exit(128)
1161
1162         self.useClientSpec = False
1163         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1164             self.useClientSpec = True
1165         if self.useClientSpec:
1166             self.clientSpecDirs = getClientSpec()
1167
1168         if self.useClientSpec:
1169             # all files are relative to the client spec
1170             self.clientPath = getClientRoot()
1171         else:
1172             self.clientPath = p4Where(self.depotPath)
1173
1174         if self.clientPath == "":
1175             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1176
1177         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1178         self.oldWorkingDirectory = os.getcwd()
1179
1180         # ensure the clientPath exists
1181         if not os.path.exists(self.clientPath):
1182             os.makedirs(self.clientPath)
1183
1184         chdir(self.clientPath)
1185         print "Synchronizing p4 checkout..."
1186         p4_sync("...")
1187         self.check()
1188
1189         commits = []
1190         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1191             commits.append(line.strip())
1192         commits.reverse()
1193
1194         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1195             self.checkAuthorship = False
1196         else:
1197             self.checkAuthorship = True
1198
1199         if self.preserveUser:
1200             self.checkValidP4Users(commits)
1201
1202         while len(commits) > 0:
1203             commit = commits[0]
1204             commits = commits[1:]
1205             self.applyCommit(commit)
1206             if not self.interactive:
1207                 break
1208
1209         if len(commits) == 0:
1210             print "All changes applied!"
1211             chdir(self.oldWorkingDirectory)
1212
1213             sync = P4Sync()
1214             sync.run([])
1215
1216             rebase = P4Rebase()
1217             rebase.rebase()
1218
1219         return True
1220
1221 class View(object):
1222     """Represent a p4 view ("p4 help views"), and map files in a
1223        repo according to the view."""
1224
1225     class Path(object):
1226         """A depot or client path, possibly containing wildcards.
1227            The only one supported is ... at the end, currently.
1228            Initialize with the full path, with //depot or //client."""
1229
1230         def __init__(self, path, is_depot):
1231             self.path = path
1232             self.is_depot = is_depot
1233             self.find_wildcards()
1234             # remember the prefix bit, useful for relative mappings
1235             m = re.match("(//[^/]+/)", self.path)
1236             if not m:
1237                 die("Path %s does not start with //prefix/" % self.path)
1238             prefix = m.group(1)
1239             if not self.is_depot:
1240                 # strip //client/ on client paths
1241                 self.path = self.path[len(prefix):]
1242
1243         def find_wildcards(self):
1244             """Make sure wildcards are valid, and set up internal
1245                variables."""
1246
1247             self.ends_triple_dot = False
1248             # There are three wildcards allowed in p4 views
1249             # (see "p4 help views").  This code knows how to
1250             # handle "..." (only at the end), but cannot deal with
1251             # "%%n" or "*".  Only check the depot_side, as p4 should
1252             # validate that the client_side matches too.
1253             if re.search(r'%%[1-9]', self.path):
1254                 die("Can't handle %%n wildcards in view: %s" % self.path)
1255             if self.path.find("*") >= 0:
1256                 die("Can't handle * wildcards in view: %s" % self.path)
1257             triple_dot_index = self.path.find("...")
1258             if triple_dot_index >= 0:
1259                 if not self.path.endswith("..."):
1260                     die("Can handle ... wildcard only at end of path: %s" %
1261                         self.path)
1262                 self.ends_triple_dot = True
1263
1264         def ensure_compatible(self, other_path):
1265             """Make sure the wildcards agree."""
1266             if self.ends_triple_dot != other_path.ends_triple_dot:
1267                  die("Both paths must end with ... if either does;\n" +
1268                      "paths: %s %s" % (self.path, other_path.path))
1269
1270         def match_wildcards(self, test_path):
1271             """See if this test_path matches us, and fill in the value
1272                of the wildcards if so.  Returns a tuple of
1273                (True|False, wildcards[]).  For now, only the ... at end
1274                is supported, so at most one wildcard."""
1275             if self.ends_triple_dot:
1276                 dotless = self.path[:-3]
1277                 if test_path.startswith(dotless):
1278                     wildcard = test_path[len(dotless):]
1279                     return (True, [ wildcard ])
1280             else:
1281                 if test_path == self.path:
1282                     return (True, [])
1283             return (False, [])
1284
1285         def match(self, test_path):
1286             """Just return if it matches; don't bother with the wildcards."""
1287             b, _ = self.match_wildcards(test_path)
1288             return b
1289
1290         def fill_in_wildcards(self, wildcards):
1291             """Return the relative path, with the wildcards filled in
1292                if there are any."""
1293             if self.ends_triple_dot:
1294                 return self.path[:-3] + wildcards[0]
1295             else:
1296                 return self.path
1297
1298     class Mapping(object):
1299         def __init__(self, depot_side, client_side, overlay, exclude):
1300             # depot_side is without the trailing /... if it had one
1301             self.depot_side = View.Path(depot_side, is_depot=True)
1302             self.client_side = View.Path(client_side, is_depot=False)
1303             self.overlay = overlay  # started with "+"
1304             self.exclude = exclude  # started with "-"
1305             assert not (self.overlay and self.exclude)
1306             self.depot_side.ensure_compatible(self.client_side)
1307
1308         def __str__(self):
1309             c = " "
1310             if self.overlay:
1311                 c = "+"
1312             if self.exclude:
1313                 c = "-"
1314             return "View.Mapping: %s%s -> %s" % \
1315                    (c, self.depot_side, self.client_side)
1316
1317         def map_depot_to_client(self, depot_path):
1318             """Calculate the client path if using this mapping on the
1319                given depot path; does not consider the effect of other
1320                mappings in a view.  Even excluded mappings are returned."""
1321             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1322             if not matches:
1323                 return ""
1324             client_path = self.client_side.fill_in_wildcards(wildcards)
1325             return client_path
1326
1327     #
1328     # View methods
1329     #
1330     def __init__(self):
1331         self.mappings = []
1332
1333     def append(self, view_line):
1334         """Parse a view line, splitting it into depot and client
1335            sides.  Append to self.mappings, preserving order."""
1336
1337         # Split the view line into exactly two words.  P4 enforces
1338         # structure on these lines that simplifies this quite a bit.
1339         #
1340         # Either or both words may be double-quoted.
1341         # Single quotes do not matter.
1342         # Double-quote marks cannot occur inside the words.
1343         # A + or - prefix is also inside the quotes.
1344         # There are no quotes unless they contain a space.
1345         # The line is already white-space stripped.
1346         # The two words are separated by a single space.
1347         #
1348         if view_line[0] == '"':
1349             # First word is double quoted.  Find its end.
1350             close_quote_index = view_line.find('"', 1)
1351             if close_quote_index <= 0:
1352                 die("No first-word closing quote found: %s" % view_line)
1353             depot_side = view_line[1:close_quote_index]
1354             # skip closing quote and space
1355             rhs_index = close_quote_index + 1 + 1
1356         else:
1357             space_index = view_line.find(" ")
1358             if space_index <= 0:
1359                 die("No word-splitting space found: %s" % view_line)
1360             depot_side = view_line[0:space_index]
1361             rhs_index = space_index + 1
1362
1363         if view_line[rhs_index] == '"':
1364             # Second word is double quoted.  Make sure there is a
1365             # double quote at the end too.
1366             if not view_line.endswith('"'):
1367                 die("View line with rhs quote should end with one: %s" %
1368                     view_line)
1369             # skip the quotes
1370             client_side = view_line[rhs_index+1:-1]
1371         else:
1372             client_side = view_line[rhs_index:]
1373
1374         # prefix + means overlay on previous mapping
1375         overlay = False
1376         if depot_side.startswith("+"):
1377             overlay = True
1378             depot_side = depot_side[1:]
1379
1380         # prefix - means exclude this path
1381         exclude = False
1382         if depot_side.startswith("-"):
1383             exclude = True
1384             depot_side = depot_side[1:]
1385
1386         m = View.Mapping(depot_side, client_side, overlay, exclude)
1387         self.mappings.append(m)
1388
1389     def map_in_client(self, depot_path):
1390         """Return the relative location in the client where this
1391            depot file should live.  Returns "" if the file should
1392            not be mapped in the client."""
1393
1394         paths_filled = []
1395         client_path = ""
1396
1397         # look at later entries first
1398         for m in self.mappings[::-1]:
1399
1400             # see where will this path end up in the client
1401             p = m.map_depot_to_client(depot_path)
1402
1403             if p == "":
1404                 # Depot path does not belong in client.  Must remember
1405                 # this, as previous items should not cause files to
1406                 # exist in this path either.  Remember that the list is
1407                 # being walked from the end, which has higher precedence.
1408                 # Overlap mappings do not exclude previous mappings.
1409                 if not m.overlay:
1410                     paths_filled.append(m.client_side)
1411
1412             else:
1413                 # This mapping matched; no need to search any further.
1414                 # But, the mapping could be rejected if the client path
1415                 # has already been claimed by an earlier mapping.
1416                 already_mapped_in_client = False
1417                 for f in paths_filled:
1418                     # this is View.Path.match
1419                     if f.match(p):
1420                         already_mapped_in_client = True
1421                         break
1422                 if not already_mapped_in_client:
1423                     # Include this file, unless it is from a line that
1424                     # explicitly said to exclude it.
1425                     if not m.exclude:
1426                         client_path = p
1427
1428                 # a match, even if rejected, always stops the search
1429                 break
1430
1431         return client_path
1432
1433 class P4Sync(Command, P4UserMap):
1434     delete_actions = ( "delete", "move/delete", "purge" )
1435
1436     def __init__(self):
1437         Command.__init__(self)
1438         P4UserMap.__init__(self)
1439         self.options = [
1440                 optparse.make_option("--branch", dest="branch"),
1441                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1442                 optparse.make_option("--changesfile", dest="changesFile"),
1443                 optparse.make_option("--silent", dest="silent", action="store_true"),
1444                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1445                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1446                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1447                                      help="Import into refs/heads/ , not refs/remotes"),
1448                 optparse.make_option("--max-changes", dest="maxChanges"),
1449                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1450                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1451                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1452                                      help="Only sync files that are included in the Perforce Client Spec")
1453         ]
1454         self.description = """Imports from Perforce into a git repository.\n
1455     example:
1456     //depot/my/project/ -- to import the current head
1457     //depot/my/project/@all -- to import everything
1458     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1459
1460     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1461
1462         self.usage += " //depot/path[@revRange]"
1463         self.silent = False
1464         self.createdBranches = set()
1465         self.committedChanges = set()
1466         self.branch = ""
1467         self.detectBranches = False
1468         self.detectLabels = False
1469         self.changesFile = ""
1470         self.syncWithOrigin = True
1471         self.verbose = False
1472         self.importIntoRemotes = True
1473         self.maxChanges = ""
1474         self.isWindows = (platform.system() == "Windows")
1475         self.keepRepoPath = False
1476         self.depotPaths = None
1477         self.p4BranchesInGit = []
1478         self.cloneExclude = []
1479         self.useClientSpec = False
1480         self.useClientSpec_from_options = False
1481         self.clientSpecDirs = None
1482
1483         if gitConfig("git-p4.syncFromOrigin") == "false":
1484             self.syncWithOrigin = False
1485
1486     #
1487     # P4 wildcards are not allowed in filenames.  P4 complains
1488     # if you simply add them, but you can force it with "-f", in
1489     # which case it translates them into %xx encoding internally.
1490     # Search for and fix just these four characters.  Do % last so
1491     # that fixing it does not inadvertently create new %-escapes.
1492     #
1493     def wildcard_decode(self, path):
1494         # Cannot have * in a filename in windows; untested as to
1495         # what p4 would do in such a case.
1496         if not self.isWindows:
1497             path = path.replace("%2A", "*")
1498         path = path.replace("%23", "#") \
1499                    .replace("%40", "@") \
1500                    .replace("%25", "%")
1501         return path
1502
1503     def extractFilesFromCommit(self, commit):
1504         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1505                              for path in self.cloneExclude]
1506         files = []
1507         fnum = 0
1508         while commit.has_key("depotFile%s" % fnum):
1509             path =  commit["depotFile%s" % fnum]
1510
1511             if [p for p in self.cloneExclude
1512                 if p4PathStartsWith(path, p)]:
1513                 found = False
1514             else:
1515                 found = [p for p in self.depotPaths
1516                          if p4PathStartsWith(path, p)]
1517             if not found:
1518                 fnum = fnum + 1
1519                 continue
1520
1521             file = {}
1522             file["path"] = path
1523             file["rev"] = commit["rev%s" % fnum]
1524             file["action"] = commit["action%s" % fnum]
1525             file["type"] = commit["type%s" % fnum]
1526             files.append(file)
1527             fnum = fnum + 1
1528         return files
1529
1530     def stripRepoPath(self, path, prefixes):
1531         if self.useClientSpec:
1532             return self.clientSpecDirs.map_in_client(path)
1533
1534         if self.keepRepoPath:
1535             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1536
1537         for p in prefixes:
1538             if p4PathStartsWith(path, p):
1539                 path = path[len(p):]
1540
1541         return path
1542
1543     def splitFilesIntoBranches(self, commit):
1544         branches = {}
1545         fnum = 0
1546         while commit.has_key("depotFile%s" % fnum):
1547             path =  commit["depotFile%s" % fnum]
1548             found = [p for p in self.depotPaths
1549                      if p4PathStartsWith(path, p)]
1550             if not found:
1551                 fnum = fnum + 1
1552                 continue
1553
1554             file = {}
1555             file["path"] = path
1556             file["rev"] = commit["rev%s" % fnum]
1557             file["action"] = commit["action%s" % fnum]
1558             file["type"] = commit["type%s" % fnum]
1559             fnum = fnum + 1
1560
1561             relPath = self.stripRepoPath(path, self.depotPaths)
1562
1563             for branch in self.knownBranches.keys():
1564
1565                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1566                 if relPath.startswith(branch + "/"):
1567                     if branch not in branches:
1568                         branches[branch] = []
1569                     branches[branch].append(file)
1570                     break
1571
1572         return branches
1573
1574     # output one file from the P4 stream
1575     # - helper for streamP4Files
1576
1577     def streamOneP4File(self, file, contents):
1578         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1579         relPath = self.wildcard_decode(relPath)
1580         if verbose:
1581             sys.stderr.write("%s\n" % relPath)
1582
1583         (type_base, type_mods) = split_p4_type(file["type"])
1584
1585         git_mode = "100644"
1586         if "x" in type_mods:
1587             git_mode = "100755"
1588         if type_base == "symlink":
1589             git_mode = "120000"
1590             # p4 print on a symlink contains "target\n"; remove the newline
1591             data = ''.join(contents)
1592             contents = [data[:-1]]
1593
1594         if type_base == "utf16":
1595             # p4 delivers different text in the python output to -G
1596             # than it does when using "print -o", or normal p4 client
1597             # operations.  utf16 is converted to ascii or utf8, perhaps.
1598             # But ascii text saved as -t utf16 is completely mangled.
1599             # Invoke print -o to get the real contents.
1600             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1601             contents = [ text ]
1602
1603         if type_base == "apple":
1604             # Apple filetype files will be streamed as a concatenation of
1605             # its appledouble header and the contents.  This is useless
1606             # on both macs and non-macs.  If using "print -q -o xx", it
1607             # will create "xx" with the data, and "%xx" with the header.
1608             # This is also not very useful.
1609             #
1610             # Ideally, someday, this script can learn how to generate
1611             # appledouble files directly and import those to git, but
1612             # non-mac machines can never find a use for apple filetype.
1613             print "\nIgnoring apple filetype file %s" % file['depotFile']
1614             return
1615
1616         # Perhaps windows wants unicode, utf16 newlines translated too;
1617         # but this is not doing it.
1618         if self.isWindows and type_base == "text":
1619             mangled = []
1620             for data in contents:
1621                 data = data.replace("\r\n", "\n")
1622                 mangled.append(data)
1623             contents = mangled
1624
1625         # Note that we do not try to de-mangle keywords on utf16 files,
1626         # even though in theory somebody may want that.
1627         if type_base in ("text", "unicode", "binary"):
1628             if "ko" in type_mods:
1629                 text = ''.join(contents)
1630                 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1631                 contents = [ text ]
1632             elif "k" in type_mods:
1633                 text = ''.join(contents)
1634                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1635                 contents = [ text ]
1636
1637         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1638
1639         # total length...
1640         length = 0
1641         for d in contents:
1642             length = length + len(d)
1643
1644         self.gitStream.write("data %d\n" % length)
1645         for d in contents:
1646             self.gitStream.write(d)
1647         self.gitStream.write("\n")
1648
1649     def streamOneP4Deletion(self, file):
1650         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1651         if verbose:
1652             sys.stderr.write("delete %s\n" % relPath)
1653         self.gitStream.write("D %s\n" % relPath)
1654
1655     # handle another chunk of streaming data
1656     def streamP4FilesCb(self, marshalled):
1657
1658         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1659             # start of a new file - output the old one first
1660             self.streamOneP4File(self.stream_file, self.stream_contents)
1661             self.stream_file = {}
1662             self.stream_contents = []
1663             self.stream_have_file_info = False
1664
1665         # pick up the new file information... for the
1666         # 'data' field we need to append to our array
1667         for k in marshalled.keys():
1668             if k == 'data':
1669                 self.stream_contents.append(marshalled['data'])
1670             else:
1671                 self.stream_file[k] = marshalled[k]
1672
1673         self.stream_have_file_info = True
1674
1675     # Stream directly from "p4 files" into "git fast-import"
1676     def streamP4Files(self, files):
1677         filesForCommit = []
1678         filesToRead = []
1679         filesToDelete = []
1680
1681         for f in files:
1682             # if using a client spec, only add the files that have
1683             # a path in the client
1684             if self.clientSpecDirs:
1685                 if self.clientSpecDirs.map_in_client(f['path']) == "":
1686                     continue
1687
1688             filesForCommit.append(f)
1689             if f['action'] in self.delete_actions:
1690                 filesToDelete.append(f)
1691             else:
1692                 filesToRead.append(f)
1693
1694         # deleted files...
1695         for f in filesToDelete:
1696             self.streamOneP4Deletion(f)
1697
1698         if len(filesToRead) > 0:
1699             self.stream_file = {}
1700             self.stream_contents = []
1701             self.stream_have_file_info = False
1702
1703             # curry self argument
1704             def streamP4FilesCbSelf(entry):
1705                 self.streamP4FilesCb(entry)
1706
1707             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1708
1709             p4CmdList(["-x", "-", "print"],
1710                       stdin=fileArgs,
1711                       cb=streamP4FilesCbSelf)
1712
1713             # do the last chunk
1714             if self.stream_file.has_key('depotFile'):
1715                 self.streamOneP4File(self.stream_file, self.stream_contents)
1716
1717     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1718         epoch = details["time"]
1719         author = details["user"]
1720         self.branchPrefixes = branchPrefixes
1721
1722         if self.verbose:
1723             print "commit into %s" % branch
1724
1725         # start with reading files; if that fails, we should not
1726         # create a commit.
1727         new_files = []
1728         for f in files:
1729             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1730                 new_files.append (f)
1731             else:
1732                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1733
1734         self.gitStream.write("commit %s\n" % branch)
1735 #        gitStream.write("mark :%s\n" % details["change"])
1736         self.committedChanges.add(int(details["change"]))
1737         committer = ""
1738         if author not in self.users:
1739             self.getUserMapFromPerforceServer()
1740         if author in self.users:
1741             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1742         else:
1743             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1744
1745         self.gitStream.write("committer %s\n" % committer)
1746
1747         self.gitStream.write("data <<EOT\n")
1748         self.gitStream.write(details["desc"])
1749         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1750                              % (','.join (branchPrefixes), details["change"]))
1751         if len(details['options']) > 0:
1752             self.gitStream.write(": options = %s" % details['options'])
1753         self.gitStream.write("]\nEOT\n\n")
1754
1755         if len(parent) > 0:
1756             if self.verbose:
1757                 print "parent %s" % parent
1758             self.gitStream.write("from %s\n" % parent)
1759
1760         self.streamP4Files(new_files)
1761         self.gitStream.write("\n")
1762
1763         change = int(details["change"])
1764
1765         if self.labels.has_key(change):
1766             label = self.labels[change]
1767             labelDetails = label[0]
1768             labelRevisions = label[1]
1769             if self.verbose:
1770                 print "Change %s is labelled %s" % (change, labelDetails)
1771
1772             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1773                                                     for p in branchPrefixes])
1774
1775             if len(files) == len(labelRevisions):
1776
1777                 cleanedFiles = {}
1778                 for info in files:
1779                     if info["action"] in self.delete_actions:
1780                         continue
1781                     cleanedFiles[info["depotFile"]] = info["rev"]
1782
1783                 if cleanedFiles == labelRevisions:
1784                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1785                     self.gitStream.write("from %s\n" % branch)
1786
1787                     owner = labelDetails["Owner"]
1788                     tagger = ""
1789                     if author in self.users:
1790                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1791                     else:
1792                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1793                     self.gitStream.write("tagger %s\n" % tagger)
1794                     self.gitStream.write("data <<EOT\n")
1795                     self.gitStream.write(labelDetails["Description"])
1796                     self.gitStream.write("EOT\n\n")
1797
1798                 else:
1799                     if not self.silent:
1800                         print ("Tag %s does not match with change %s: files do not match."
1801                                % (labelDetails["label"], change))
1802
1803             else:
1804                 if not self.silent:
1805                     print ("Tag %s does not match with change %s: file count is different."
1806                            % (labelDetails["label"], change))
1807
1808     def getLabels(self):
1809         self.labels = {}
1810
1811         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1812         if len(l) > 0 and not self.silent:
1813             print "Finding files belonging to labels in %s" % `self.depotPaths`
1814
1815         for output in l:
1816             label = output["label"]
1817             revisions = {}
1818             newestChange = 0
1819             if self.verbose:
1820                 print "Querying files for label %s" % label
1821             for file in p4CmdList(["files"] +
1822                                       ["%s...@%s" % (p, label)
1823                                           for p in self.depotPaths]):
1824                 revisions[file["depotFile"]] = file["rev"]
1825                 change = int(file["change"])
1826                 if change > newestChange:
1827                     newestChange = change
1828
1829             self.labels[newestChange] = [output, revisions]
1830
1831         if self.verbose:
1832             print "Label changes: %s" % self.labels.keys()
1833
1834     def guessProjectName(self):
1835         for p in self.depotPaths:
1836             if p.endswith("/"):
1837                 p = p[:-1]
1838             p = p[p.strip().rfind("/") + 1:]
1839             if not p.endswith("/"):
1840                p += "/"
1841             return p
1842
1843     def getBranchMapping(self):
1844         lostAndFoundBranches = set()
1845
1846         user = gitConfig("git-p4.branchUser")
1847         if len(user) > 0:
1848             command = "branches -u %s" % user
1849         else:
1850             command = "branches"
1851
1852         for info in p4CmdList(command):
1853             details = p4Cmd("branch -o %s" % info["branch"])
1854             viewIdx = 0
1855             while details.has_key("View%s" % viewIdx):
1856                 paths = details["View%s" % viewIdx].split(" ")
1857                 viewIdx = viewIdx + 1
1858                 # require standard //depot/foo/... //depot/bar/... mapping
1859                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1860                     continue
1861                 source = paths[0]
1862                 destination = paths[1]
1863                 ## HACK
1864                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1865                     source = source[len(self.depotPaths[0]):-4]
1866                     destination = destination[len(self.depotPaths[0]):-4]
1867
1868                     if destination in self.knownBranches:
1869                         if not self.silent:
1870                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1871                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1872                         continue
1873
1874                     self.knownBranches[destination] = source
1875
1876                     lostAndFoundBranches.discard(destination)
1877
1878                     if source not in self.knownBranches:
1879                         lostAndFoundBranches.add(source)
1880
1881         # Perforce does not strictly require branches to be defined, so we also
1882         # check git config for a branch list.
1883         #
1884         # Example of branch definition in git config file:
1885         # [git-p4]
1886         #   branchList=main:branchA
1887         #   branchList=main:branchB
1888         #   branchList=branchA:branchC
1889         configBranches = gitConfigList("git-p4.branchList")
1890         for branch in configBranches:
1891             if branch:
1892                 (source, destination) = branch.split(":")
1893                 self.knownBranches[destination] = source
1894
1895                 lostAndFoundBranches.discard(destination)
1896
1897                 if source not in self.knownBranches:
1898                     lostAndFoundBranches.add(source)
1899
1900
1901         for branch in lostAndFoundBranches:
1902             self.knownBranches[branch] = branch
1903
1904     def getBranchMappingFromGitBranches(self):
1905         branches = p4BranchesInGit(self.importIntoRemotes)
1906         for branch in branches.keys():
1907             if branch == "master":
1908                 branch = "main"
1909             else:
1910                 branch = branch[len(self.projectName):]
1911             self.knownBranches[branch] = branch
1912
1913     def listExistingP4GitBranches(self):
1914         # branches holds mapping from name to commit
1915         branches = p4BranchesInGit(self.importIntoRemotes)
1916         self.p4BranchesInGit = branches.keys()
1917         for branch in branches.keys():
1918             self.initialParents[self.refPrefix + branch] = branches[branch]
1919
1920     def updateOptionDict(self, d):
1921         option_keys = {}
1922         if self.keepRepoPath:
1923             option_keys['keepRepoPath'] = 1
1924
1925         d["options"] = ' '.join(sorted(option_keys.keys()))
1926
1927     def readOptions(self, d):
1928         self.keepRepoPath = (d.has_key('options')
1929                              and ('keepRepoPath' in d['options']))
1930
1931     def gitRefForBranch(self, branch):
1932         if branch == "main":
1933             return self.refPrefix + "master"
1934
1935         if len(branch) <= 0:
1936             return branch
1937
1938         return self.refPrefix + self.projectName + branch
1939
1940     def gitCommitByP4Change(self, ref, change):
1941         if self.verbose:
1942             print "looking in ref " + ref + " for change %s using bisect..." % change
1943
1944         earliestCommit = ""
1945         latestCommit = parseRevision(ref)
1946
1947         while True:
1948             if self.verbose:
1949                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1950             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1951             if len(next) == 0:
1952                 if self.verbose:
1953                     print "argh"
1954                 return ""
1955             log = extractLogMessageFromGitCommit(next)
1956             settings = extractSettingsGitLog(log)
1957             currentChange = int(settings['change'])
1958             if self.verbose:
1959                 print "current change %s" % currentChange
1960
1961             if currentChange == change:
1962                 if self.verbose:
1963                     print "found %s" % next
1964                 return next
1965
1966             if currentChange < change:
1967                 earliestCommit = "^%s" % next
1968             else:
1969                 latestCommit = "%s" % next
1970
1971         return ""
1972
1973     def importNewBranch(self, branch, maxChange):
1974         # make fast-import flush all changes to disk and update the refs using the checkpoint
1975         # command so that we can try to find the branch parent in the git history
1976         self.gitStream.write("checkpoint\n\n");
1977         self.gitStream.flush();
1978         branchPrefix = self.depotPaths[0] + branch + "/"
1979         range = "@1,%s" % maxChange
1980         #print "prefix" + branchPrefix
1981         changes = p4ChangesForPaths([branchPrefix], range)
1982         if len(changes) <= 0:
1983             return False
1984         firstChange = changes[0]
1985         #print "first change in branch: %s" % firstChange
1986         sourceBranch = self.knownBranches[branch]
1987         sourceDepotPath = self.depotPaths[0] + sourceBranch
1988         sourceRef = self.gitRefForBranch(sourceBranch)
1989         #print "source " + sourceBranch
1990
1991         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1992         #print "branch parent: %s" % branchParentChange
1993         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1994         if len(gitParent) > 0:
1995             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1996             #print "parent git commit: %s" % gitParent
1997
1998         self.importChanges(changes)
1999         return True
2000
2001     def importChanges(self, changes):
2002         cnt = 1
2003         for change in changes:
2004             description = p4Cmd("describe %s" % change)
2005             self.updateOptionDict(description)
2006
2007             if not self.silent:
2008                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2009                 sys.stdout.flush()
2010             cnt = cnt + 1
2011
2012             try:
2013                 if self.detectBranches:
2014                     branches = self.splitFilesIntoBranches(description)
2015                     for branch in branches.keys():
2016                         ## HACK  --hwn
2017                         branchPrefix = self.depotPaths[0] + branch + "/"
2018
2019                         parent = ""
2020
2021                         filesForCommit = branches[branch]
2022
2023                         if self.verbose:
2024                             print "branch is %s" % branch
2025
2026                         self.updatedBranches.add(branch)
2027
2028                         if branch not in self.createdBranches:
2029                             self.createdBranches.add(branch)
2030                             parent = self.knownBranches[branch]
2031                             if parent == branch:
2032                                 parent = ""
2033                             else:
2034                                 fullBranch = self.projectName + branch
2035                                 if fullBranch not in self.p4BranchesInGit:
2036                                     if not self.silent:
2037                                         print("\n    Importing new branch %s" % fullBranch);
2038                                     if self.importNewBranch(branch, change - 1):
2039                                         parent = ""
2040                                         self.p4BranchesInGit.append(fullBranch)
2041                                     if not self.silent:
2042                                         print("\n    Resuming with change %s" % change);
2043
2044                                 if self.verbose:
2045                                     print "parent determined through known branches: %s" % parent
2046
2047                         branch = self.gitRefForBranch(branch)
2048                         parent = self.gitRefForBranch(parent)
2049
2050                         if self.verbose:
2051                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2052
2053                         if len(parent) == 0 and branch in self.initialParents:
2054                             parent = self.initialParents[branch]
2055                             del self.initialParents[branch]
2056
2057                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2058                 else:
2059                     files = self.extractFilesFromCommit(description)
2060                     self.commit(description, files, self.branch, self.depotPaths,
2061                                 self.initialParent)
2062                     self.initialParent = ""
2063             except IOError:
2064                 print self.gitError.read()
2065                 sys.exit(1)
2066
2067     def importHeadRevision(self, revision):
2068         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2069
2070         details = {}
2071         details["user"] = "git perforce import user"
2072         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2073                            % (' '.join(self.depotPaths), revision))
2074         details["change"] = revision
2075         newestRevision = 0
2076
2077         fileCnt = 0
2078         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2079
2080         for info in p4CmdList(["files"] + fileArgs):
2081
2082             if 'code' in info and info['code'] == 'error':
2083                 sys.stderr.write("p4 returned an error: %s\n"
2084                                  % info['data'])
2085                 if info['data'].find("must refer to client") >= 0:
2086                     sys.stderr.write("This particular p4 error is misleading.\n")
2087                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2088                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2089                 sys.exit(1)
2090             if 'p4ExitCode' in info:
2091                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2092                 sys.exit(1)
2093
2094
2095             change = int(info["change"])
2096             if change > newestRevision:
2097                 newestRevision = change
2098
2099             if info["action"] in self.delete_actions:
2100                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2101                 #fileCnt = fileCnt + 1
2102                 continue
2103
2104             for prop in ["depotFile", "rev", "action", "type" ]:
2105                 details["%s%s" % (prop, fileCnt)] = info[prop]
2106
2107             fileCnt = fileCnt + 1
2108
2109         details["change"] = newestRevision
2110
2111         # Use time from top-most change so that all git-p4 clones of
2112         # the same p4 repo have the same commit SHA1s.
2113         res = p4CmdList("describe -s %d" % newestRevision)
2114         newestTime = None
2115         for r in res:
2116             if r.has_key('time'):
2117                 newestTime = int(r['time'])
2118         if newestTime is None:
2119             die("\"describe -s\" on newest change %d did not give a time")
2120         details["time"] = newestTime
2121
2122         self.updateOptionDict(details)
2123         try:
2124             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2125         except IOError:
2126             print "IO error with git fast-import. Is your git version recent enough?"
2127             print self.gitError.read()
2128
2129
2130     def run(self, args):
2131         self.depotPaths = []
2132         self.changeRange = ""
2133         self.initialParent = ""
2134         self.previousDepotPaths = []
2135
2136         # map from branch depot path to parent branch
2137         self.knownBranches = {}
2138         self.initialParents = {}
2139         self.hasOrigin = originP4BranchesExist()
2140         if not self.syncWithOrigin:
2141             self.hasOrigin = False
2142
2143         if self.importIntoRemotes:
2144             self.refPrefix = "refs/remotes/p4/"
2145         else:
2146             self.refPrefix = "refs/heads/p4/"
2147
2148         if self.syncWithOrigin and self.hasOrigin:
2149             if not self.silent:
2150                 print "Syncing with origin first by calling git fetch origin"
2151             system("git fetch origin")
2152
2153         if len(self.branch) == 0:
2154             self.branch = self.refPrefix + "master"
2155             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2156                 system("git update-ref %s refs/heads/p4" % self.branch)
2157                 system("git branch -D p4");
2158             # create it /after/ importing, when master exists
2159             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2160                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2161
2162         # accept either the command-line option, or the configuration variable
2163         if self.useClientSpec:
2164             # will use this after clone to set the variable
2165             self.useClientSpec_from_options = True
2166         else:
2167             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2168                 self.useClientSpec = True
2169         if self.useClientSpec:
2170             self.clientSpecDirs = getClientSpec()
2171
2172         # TODO: should always look at previous commits,
2173         # merge with previous imports, if possible.
2174         if args == []:
2175             if self.hasOrigin:
2176                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2177             self.listExistingP4GitBranches()
2178
2179             if len(self.p4BranchesInGit) > 1:
2180                 if not self.silent:
2181                     print "Importing from/into multiple branches"
2182                 self.detectBranches = True
2183
2184             if self.verbose:
2185                 print "branches: %s" % self.p4BranchesInGit
2186
2187             p4Change = 0
2188             for branch in self.p4BranchesInGit:
2189                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2190
2191                 settings = extractSettingsGitLog(logMsg)
2192
2193                 self.readOptions(settings)
2194                 if (settings.has_key('depot-paths')
2195                     and settings.has_key ('change')):
2196                     change = int(settings['change']) + 1
2197                     p4Change = max(p4Change, change)
2198
2199                     depotPaths = sorted(settings['depot-paths'])
2200                     if self.previousDepotPaths == []:
2201                         self.previousDepotPaths = depotPaths
2202                     else:
2203                         paths = []
2204                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2205                             prev_list = prev.split("/")
2206                             cur_list = cur.split("/")
2207                             for i in range(0, min(len(cur_list), len(prev_list))):
2208                                 if cur_list[i] <> prev_list[i]:
2209                                     i = i - 1
2210                                     break
2211
2212                             paths.append ("/".join(cur_list[:i + 1]))
2213
2214                         self.previousDepotPaths = paths
2215
2216             if p4Change > 0:
2217                 self.depotPaths = sorted(self.previousDepotPaths)
2218                 self.changeRange = "@%s,#head" % p4Change
2219                 if not self.detectBranches:
2220                     self.initialParent = parseRevision(self.branch)
2221                 if not self.silent and not self.detectBranches:
2222                     print "Performing incremental import into %s git branch" % self.branch
2223
2224         if not self.branch.startswith("refs/"):
2225             self.branch = "refs/heads/" + self.branch
2226
2227         if len(args) == 0 and self.depotPaths:
2228             if not self.silent:
2229                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2230         else:
2231             if self.depotPaths and self.depotPaths != args:
2232                 print ("previous import used depot path %s and now %s was specified. "
2233                        "This doesn't work!" % (' '.join (self.depotPaths),
2234                                                ' '.join (args)))
2235                 sys.exit(1)
2236
2237             self.depotPaths = sorted(args)
2238
2239         revision = ""
2240         self.users = {}
2241
2242         # Make sure no revision specifiers are used when --changesfile
2243         # is specified.
2244         bad_changesfile = False
2245         if len(self.changesFile) > 0:
2246             for p in self.depotPaths:
2247                 if p.find("@") >= 0 or p.find("#") >= 0:
2248                     bad_changesfile = True
2249                     break
2250         if bad_changesfile:
2251             die("Option --changesfile is incompatible with revision specifiers")
2252
2253         newPaths = []
2254         for p in self.depotPaths:
2255             if p.find("@") != -1:
2256                 atIdx = p.index("@")
2257                 self.changeRange = p[atIdx:]
2258                 if self.changeRange == "@all":
2259                     self.changeRange = ""
2260                 elif ',' not in self.changeRange:
2261                     revision = self.changeRange
2262                     self.changeRange = ""
2263                 p = p[:atIdx]
2264             elif p.find("#") != -1:
2265                 hashIdx = p.index("#")
2266                 revision = p[hashIdx:]
2267                 p = p[:hashIdx]
2268             elif self.previousDepotPaths == []:
2269                 # pay attention to changesfile, if given, else import
2270                 # the entire p4 tree at the head revision
2271                 if len(self.changesFile) == 0:
2272                     revision = "#head"
2273
2274             p = re.sub ("\.\.\.$", "", p)
2275             if not p.endswith("/"):
2276                 p += "/"
2277
2278             newPaths.append(p)
2279
2280         self.depotPaths = newPaths
2281
2282
2283         self.loadUserMapFromCache()
2284         self.labels = {}
2285         if self.detectLabels:
2286             self.getLabels();
2287
2288         if self.detectBranches:
2289             ## FIXME - what's a P4 projectName ?
2290             self.projectName = self.guessProjectName()
2291
2292             if self.hasOrigin:
2293                 self.getBranchMappingFromGitBranches()
2294             else:
2295                 self.getBranchMapping()
2296             if self.verbose:
2297                 print "p4-git branches: %s" % self.p4BranchesInGit
2298                 print "initial parents: %s" % self.initialParents
2299             for b in self.p4BranchesInGit:
2300                 if b != "master":
2301
2302                     ## FIXME
2303                     b = b[len(self.projectName):]
2304                 self.createdBranches.add(b)
2305
2306         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2307
2308         importProcess = subprocess.Popen(["git", "fast-import"],
2309                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2310                                          stderr=subprocess.PIPE);
2311         self.gitOutput = importProcess.stdout
2312         self.gitStream = importProcess.stdin
2313         self.gitError = importProcess.stderr
2314
2315         if revision:
2316             self.importHeadRevision(revision)
2317         else:
2318             changes = []
2319
2320             if len(self.changesFile) > 0:
2321                 output = open(self.changesFile).readlines()
2322                 changeSet = set()
2323                 for line in output:
2324                     changeSet.add(int(line))
2325
2326                 for change in changeSet:
2327                     changes.append(change)
2328
2329                 changes.sort()
2330             else:
2331                 # catch "git-p4 sync" with no new branches, in a repo that
2332                 # does not have any existing git-p4 branches
2333                 if len(args) == 0 and not self.p4BranchesInGit:
2334                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2335                 if self.verbose:
2336                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2337                                                               self.changeRange)
2338                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2339
2340                 if len(self.maxChanges) > 0:
2341                     changes = changes[:min(int(self.maxChanges), len(changes))]
2342
2343             if len(changes) == 0:
2344                 if not self.silent:
2345                     print "No changes to import!"
2346                 return True
2347
2348             if not self.silent and not self.detectBranches:
2349                 print "Import destination: %s" % self.branch
2350
2351             self.updatedBranches = set()
2352
2353             self.importChanges(changes)
2354
2355             if not self.silent:
2356                 print ""
2357                 if len(self.updatedBranches) > 0:
2358                     sys.stdout.write("Updated branches: ")
2359                     for b in self.updatedBranches:
2360                         sys.stdout.write("%s " % b)
2361                     sys.stdout.write("\n")
2362
2363         self.gitStream.close()
2364         if importProcess.wait() != 0:
2365             die("fast-import failed: %s" % self.gitError.read())
2366         self.gitOutput.close()
2367         self.gitError.close()
2368
2369         return True
2370
2371 class P4Rebase(Command):
2372     def __init__(self):
2373         Command.__init__(self)
2374         self.options = [ ]
2375         self.description = ("Fetches the latest revision from perforce and "
2376                             + "rebases the current work (branch) against it")
2377         self.verbose = False
2378
2379     def run(self, args):
2380         sync = P4Sync()
2381         sync.run([])
2382
2383         return self.rebase()
2384
2385     def rebase(self):
2386         if os.system("git update-index --refresh") != 0:
2387             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2388         if len(read_pipe("git diff-index HEAD --")) > 0:
2389             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2390
2391         [upstream, settings] = findUpstreamBranchPoint()
2392         if len(upstream) == 0:
2393             die("Cannot find upstream branchpoint for rebase")
2394
2395         # the branchpoint may be p4/foo~3, so strip off the parent
2396         upstream = re.sub("~[0-9]+$", "", upstream)
2397
2398         print "Rebasing the current branch onto %s" % upstream
2399         oldHead = read_pipe("git rev-parse HEAD").strip()
2400         system("git rebase %s" % upstream)
2401         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2402         return True
2403
2404 class P4Clone(P4Sync):
2405     def __init__(self):
2406         P4Sync.__init__(self)
2407         self.description = "Creates a new git repository and imports from Perforce into it"
2408         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2409         self.options += [
2410             optparse.make_option("--destination", dest="cloneDestination",
2411                                  action='store', default=None,
2412                                  help="where to leave result of the clone"),
2413             optparse.make_option("-/", dest="cloneExclude",
2414                                  action="append", type="string",
2415                                  help="exclude depot path"),
2416             optparse.make_option("--bare", dest="cloneBare",
2417                                  action="store_true", default=False),
2418         ]
2419         self.cloneDestination = None
2420         self.needsGit = False
2421         self.cloneBare = False
2422
2423     # This is required for the "append" cloneExclude action
2424     def ensure_value(self, attr, value):
2425         if not hasattr(self, attr) or getattr(self, attr) is None:
2426             setattr(self, attr, value)
2427         return getattr(self, attr)
2428
2429     def defaultDestination(self, args):
2430         ## TODO: use common prefix of args?
2431         depotPath = args[0]
2432         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2433         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2434         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2435         depotDir = re.sub(r"/$", "", depotDir)
2436         return os.path.split(depotDir)[1]
2437
2438     def run(self, args):
2439         if len(args) < 1:
2440             return False
2441
2442         if self.keepRepoPath and not self.cloneDestination:
2443             sys.stderr.write("Must specify destination for --keep-path\n")
2444             sys.exit(1)
2445
2446         depotPaths = args
2447
2448         if not self.cloneDestination and len(depotPaths) > 1:
2449             self.cloneDestination = depotPaths[-1]
2450             depotPaths = depotPaths[:-1]
2451
2452         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2453         for p in depotPaths:
2454             if not p.startswith("//"):
2455                 return False
2456
2457         if not self.cloneDestination:
2458             self.cloneDestination = self.defaultDestination(args)
2459
2460         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2461
2462         if not os.path.exists(self.cloneDestination):
2463             os.makedirs(self.cloneDestination)
2464         chdir(self.cloneDestination)
2465
2466         init_cmd = [ "git", "init" ]
2467         if self.cloneBare:
2468             init_cmd.append("--bare")
2469         subprocess.check_call(init_cmd)
2470
2471         if not P4Sync.run(self, depotPaths):
2472             return False
2473         if self.branch != "master":
2474             if self.importIntoRemotes:
2475                 masterbranch = "refs/remotes/p4/master"
2476             else:
2477                 masterbranch = "refs/heads/p4/master"
2478             if gitBranchExists(masterbranch):
2479                 system("git branch master %s" % masterbranch)
2480                 if not self.cloneBare:
2481                     system("git checkout -f")
2482             else:
2483                 print "Could not detect main branch. No checkout/master branch created."
2484
2485         # auto-set this variable if invoked with --use-client-spec
2486         if self.useClientSpec_from_options:
2487             system("git config --bool git-p4.useclientspec true")
2488
2489         return True
2490
2491 class P4Branches(Command):
2492     def __init__(self):
2493         Command.__init__(self)
2494         self.options = [ ]
2495         self.description = ("Shows the git branches that hold imports and their "
2496                             + "corresponding perforce depot paths")
2497         self.verbose = False
2498
2499     def run(self, args):
2500         if originP4BranchesExist():
2501             createOrUpdateBranchesFromOrigin()
2502
2503         cmdline = "git rev-parse --symbolic "
2504         cmdline += " --remotes"
2505
2506         for line in read_pipe_lines(cmdline):
2507             line = line.strip()
2508
2509             if not line.startswith('p4/') or line == "p4/HEAD":
2510                 continue
2511             branch = line
2512
2513             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2514             settings = extractSettingsGitLog(log)
2515
2516             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2517         return True
2518
2519 class HelpFormatter(optparse.IndentedHelpFormatter):
2520     def __init__(self):
2521         optparse.IndentedHelpFormatter.__init__(self)
2522
2523     def format_description(self, description):
2524         if description:
2525             return description + "\n"
2526         else:
2527             return ""
2528
2529 def printUsage(commands):
2530     print "usage: %s <command> [options]" % sys.argv[0]
2531     print ""
2532     print "valid commands: %s" % ", ".join(commands)
2533     print ""
2534     print "Try %s <command> --help for command specific help." % sys.argv[0]
2535     print ""
2536
2537 commands = {
2538     "debug" : P4Debug,
2539     "submit" : P4Submit,
2540     "commit" : P4Submit,
2541     "sync" : P4Sync,
2542     "rebase" : P4Rebase,
2543     "clone" : P4Clone,
2544     "rollback" : P4RollBack,
2545     "branches" : P4Branches
2546 }
2547
2548
2549 def main():
2550     if len(sys.argv[1:]) == 0:
2551         printUsage(commands.keys())
2552         sys.exit(2)
2553
2554     cmd = ""
2555     cmdName = sys.argv[1]
2556     try:
2557         klass = commands[cmdName]
2558         cmd = klass()
2559     except KeyError:
2560         print "unknown command %s" % cmdName
2561         print ""
2562         printUsage(commands.keys())
2563         sys.exit(2)
2564
2565     options = cmd.options
2566     cmd.gitdir = os.environ.get("GIT_DIR", None)
2567
2568     args = sys.argv[2:]
2569
2570     if len(options) > 0:
2571         if cmd.needsGit:
2572             options.append(optparse.make_option("--git-dir", dest="gitdir"))
2573
2574         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2575                                        options,
2576                                        description = cmd.description,
2577                                        formatter = HelpFormatter())
2578
2579         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2580     global verbose
2581     verbose = cmd.verbose
2582     if cmd.needsGit:
2583         if cmd.gitdir == None:
2584             cmd.gitdir = os.path.abspath(".git")
2585             if not isValidGitDir(cmd.gitdir):
2586                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2587                 if os.path.exists(cmd.gitdir):
2588                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2589                     if len(cdup) > 0:
2590                         chdir(cdup);
2591
2592         if not isValidGitDir(cmd.gitdir):
2593             if isValidGitDir(cmd.gitdir + "/.git"):
2594                 cmd.gitdir += "/.git"
2595             else:
2596                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2597
2598         os.environ["GIT_DIR"] = cmd.gitdir
2599
2600     if not cmd.run(args):
2601         parser.print_help()
2602         sys.exit(2)
2603
2604
2605 if __name__ == '__main__':
2606     main()