p4-fast-export.py

   1 #!/usr/bin/python
   2 #
   3 # p4-fast-export.py
   4 #
   5 # Author: Simon Hausmann <hausmann@kde.org>
   6 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   7 #
   8 # TODO:
   9 #       - support integrations (at least p4i)
  10 #       - support p4 submit (hah!)
  11 #       - emulate p4's delete behavior: if a directory becomes empty delete it. continue
  12 #         with parent dir until non-empty dir is found.
  13 #
  14 import os, string, sys, time, os.path
  15 import marshal, popen2, getopt, sha
  16 from sets import Set;
  17
  18 cacheDebug = False
  19
  20 silent = False
  21 knownBranches = Set()
  22 createdBranches = Set()
  23 committedChanges = Set()
  24 branch = "refs/heads/master"
  25 globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read()
  26 detectBranches = False
  27 changesFile = ""
  28 if len(globalPrefix) != 0:
  29     globalPrefix = globalPrefix[:-1]
  30
  31 try:
  32     opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent", "known-branches=" ])
  33 except getopt.GetoptError:
  34     print "fixme, syntax error"
  35     sys.exit(1)
  36
  37 for o, a in opts:
  38     if o == "--branch":
  39         branch = "refs/heads/" + a
  40     elif o == "--detect-branches":
  41         detectBranches = True
  42     elif o == "--changesfile":
  43         changesFile = a
  44     elif o == "--silent":
  45         silent= True
  46     elif o == "--known-branches":
  47         for branch in open(a).readlines():
  48             knownBranches.add(branch[:-1])
  49
  50 if len(args) == 0 and len(globalPrefix) != 0:
  51     if not silent:
  52         print "[using previously specified depot path %s]" % globalPrefix
  53 elif len(args) != 1:
  54     print "usage: %s //depot/path[@revRange]" % sys.argv[0]
  55     print "\n    example:"
  56     print "    %s //depot/my/project/ -- to import the current head"
  57     print "    %s //depot/my/project/@all -- to import everything"
  58     print "    %s //depot/my/project/@1,6 -- to import only from revision 1 to 6"
  59     print ""
  60     print "    (a ... is not needed in the path p4 specification, it's added implicitly)"
  61     print ""
  62     sys.exit(1)
  63 else:
  64     if len(globalPrefix) != 0 and globalPrefix != args[0]:
  65         print "previous import used depot path %s and now %s was specified. this doesn't work!" % (globalPrefix, args[0])
  66         sys.exit(1)
  67     globalPrefix = args[0]
  68
  69 changeRange = ""
  70 revision = ""
  71 users = {}
  72 initialParent = ""
  73 lastChange = 0
  74 initialTag = ""
  75
  76 if globalPrefix.find("@") != -1:
  77     atIdx = globalPrefix.index("@")
  78     changeRange = globalPrefix[atIdx:]
  79     if changeRange == "@all":
  80         changeRange = ""
  81     elif changeRange.find(",") == -1:
  82         revision = changeRange
  83         changeRange = ""
  84     globalPrefix = globalPrefix[0:atIdx]
  85 elif globalPrefix.find("#") != -1:
  86     hashIdx = globalPrefix.index("#")
  87     revision = globalPrefix[hashIdx:]
  88     globalPrefix = globalPrefix[0:hashIdx]
  89 elif len(previousDepotPath) == 0:
  90     revision = "#head"
  91
  92 if globalPrefix.endswith("..."):
  93     globalPrefix = globalPrefix[:-3]
  94
  95 if not globalPrefix.endswith("/"):
  96     globalPrefix += "/"
  97
  98 def p4File(depotPath):
  99     cacheKey = "/tmp/p4cache/data-" + sha.new(depotPath).hexdigest()
 100
 101     data = 0
 102     try:
 103         if not cacheDebug:
 104             raise
 105         data = open(cacheKey, "rb").read()
 106     except:
 107         data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 108         if cacheDebug:
 109             open(cacheKey, "wb").write(data)
 110
 111     return data
 112
 113 def p4CmdList(cmd):
 114     fullCmd = "p4 -G %s" % cmd;
 115
 116     cacheKey = sha.new(fullCmd).hexdigest()
 117     cacheKey = "/tmp/p4cache/cmd-" + cacheKey
 118
 119     cached = True
 120     pipe = 0
 121     try:
 122         if not cacheDebug:
 123             raise
 124         pipe = open(cacheKey, "rb")
 125     except:
 126         cached = False
 127         pipe = os.popen(fullCmd, "rb")
 128
 129     result = []
 130     try:
 131         while True:
 132             entry = marshal.load(pipe)
 133             result.append(entry)
 134     except EOFError:
 135         pass
 136     pipe.close()
 137
 138     if not cached and cacheDebug:
 139         pipe = open(cacheKey, "wb")
 140         for r in result:
 141             marshal.dump(r, pipe)
 142         pipe.close()
 143
 144     return result
 145
 146 def p4Cmd(cmd):
 147     list = p4CmdList(cmd)
 148     result = {}
 149     for entry in list:
 150         result.update(entry)
 151     return result;
 152
 153 def extractFilesFromCommit(commit):
 154     files = []
 155     fnum = 0
 156     while commit.has_key("depotFile%s" % fnum):
 157         path =  commit["depotFile%s" % fnum]
 158         if not path.startswith(globalPrefix):
 159 #            if not silent:
 160 #                print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change)
 161             fnum = fnum + 1
 162             continue
 163
 164         file = {}
 165         file["path"] = path
 166         file["rev"] = commit["rev%s" % fnum]
 167         file["action"] = commit["action%s" % fnum]
 168         file["type"] = commit["type%s" % fnum]
 169         files.append(file)
 170         fnum = fnum + 1
 171     return files
 172
 173 def isSubPathOf(first, second):
 174     if not first.startswith(second):
 175         return False
 176     if first == second:
 177         return True
 178     return first[len(second)] == "/"
 179
 180 def branchesForCommit(files):
 181     global knownBranches
 182     branches = Set()
 183
 184     for file in files:
 185         relativePath = file["path"][len(globalPrefix):]
 186         # strip off the filename
 187         relativePath = relativePath[0:relativePath.rfind("/")]
 188
 189 #        if len(branches) == 0:
 190 #            branches.add(relativePath)
 191 #            knownBranches.add(relativePath)
 192 #            continue
 193
 194         ###### this needs more testing :)
 195         knownBranch = False
 196         for branch in branches:
 197             if relativePath == branch:
 198                 knownBranch = True
 199                 break
 200 #            if relativePath.startswith(branch):
 201             if isSubPathOf(relativePath, branch):
 202                 knownBranch = True
 203                 break
 204 #            if branch.startswith(relativePath):
 205             if isSubPathOf(branch, relativePath):
 206                 branches.remove(branch)
 207                 break
 208
 209         if knownBranch:
 210             continue
 211
 212         for branch in knownBranches:
 213             #if relativePath.startswith(branch):
 214             if isSubPathOf(relativePath, branch):
 215                 if len(branches) == 0:
 216                     relativePath = branch
 217                 else:
 218                     knownBranch = True
 219                 break
 220
 221         if knownBranch:
 222             continue
 223
 224         branches.add(relativePath)
 225         knownBranches.add(relativePath)
 226
 227     return branches
 228
 229 def findBranchParent(branchPrefix, files):
 230     for file in files:
 231         path = file["path"]
 232         if not path.startswith(branchPrefix):
 233             continue
 234         action = file["action"]
 235         if action != "integrate" and action != "branch":
 236             continue
 237         rev = file["rev"]
 238         depotPath = path + "#" + rev
 239
 240         log = p4CmdList("filelog \"%s\"" % depotPath)
 241         if len(log) != 1:
 242             print "eek! I got confused by the filelog of %s" % depotPath
 243             sys.exit(1);
 244
 245         log = log[0]
 246         if log["action0"] != action:
 247             print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 248             sys.exit(1);
 249
 250         branchAction = log["how0,0"]
 251 #        if branchAction == "branch into" or branchAction == "ignored":
 252 #            continue # ignore for branching
 253
 254         if not branchAction.endswith(" from"):
 255             continue # ignore for branching
 256 #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 257 #            sys.exit(1);
 258
 259         source = log["file0,0"]
 260         if source.startswith(branchPrefix):
 261             continue
 262
 263         lastSourceRev = log["erev0,0"]
 264
 265         sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 266         if len(sourceLog) != 1:
 267             print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 268             sys.exit(1);
 269         sourceLog = sourceLog[0]
 270
 271         relPath = source[len(globalPrefix):]
 272         # strip off the filename
 273         relPath = relPath[0:relPath.rfind("/")]
 274
 275         for branch in knownBranches:
 276             if isSubPathOf(relPath, branch):
 277 #                print "determined parent branch branch %s due to change in file %s" % (branch, source)
 278                 return "refs/heads/%s" % branch
 279 #            else:
 280 #                print "%s is not a subpath of branch %s" % (relPath, branch)
 281
 282     return ""
 283
 284 def commit(details, files, branch, branchPrefix, parent):
 285     global users
 286     global lastChange
 287     global committedChanges
 288
 289     epoch = details["time"]
 290     author = details["user"]
 291
 292     gitStream.write("commit %s\n" % branch)
 293     gitStream.write("mark :%s\n" % details["change"])
 294     committedChanges.add(int(details["change"]))
 295     committer = ""
 296     if author in users:
 297         committer = "%s %s %s" % (users[author], epoch, tz)
 298     else:
 299         committer = "%s <a@b> %s %s" % (author, epoch, tz)
 300
 301     gitStream.write("committer %s\n" % committer)
 302
 303     gitStream.write("data <<EOT\n")
 304     gitStream.write(details["desc"])
 305     gitStream.write("\n[ imported from %s; change %s ]\n" % (branchPrefix, details["change"]))
 306     gitStream.write("EOT\n\n")
 307
 308     if len(parent) > 0:
 309         gitStream.write("from %s\n" % parent)
 310
 311     for file in files:
 312         path = file["path"]
 313         if not path.startswith(branchPrefix):
 314 #            if not silent:
 315 #                print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 316             continue
 317         rev = file["rev"]
 318         depotPath = path + "#" + rev
 319         relPath = path[len(branchPrefix):]
 320         action = file["action"]
 321
 322         if action == "delete":
 323             gitStream.write("D %s\n" % relPath)
 324         else:
 325             mode = 644
 326             if file["type"].startswith("x"):
 327                 mode = 755
 328
 329             data = p4File(depotPath)
 330
 331             gitStream.write("M %s inline %s\n" % (mode, relPath))
 332             gitStream.write("data %s\n" % len(data))
 333             gitStream.write(data)
 334             gitStream.write("\n")
 335
 336     gitStream.write("\n")
 337
 338     lastChange = int(details["change"])
 339
 340 def getUserMap():
 341     users = {}
 342
 343     for output in p4CmdList("users"):
 344         if not output.has_key("User"):
 345             continue
 346         users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 347     return users
 348
 349 users = getUserMap()
 350
 351 if len(changeRange) == 0:
 352     try:
 353         sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
 354         output = sout.read()
 355         if output.endswith("\n"):
 356             output = output[:-1]
 357         tagIdx = output.index(" tags/p4/")
 358         caretIdx = output.find("^")
 359         endPos = len(output)
 360         if caretIdx != -1:
 361             endPos = caretIdx
 362         rev = int(output[tagIdx + 9 : endPos]) + 1
 363         changeRange = "@%s,#head" % rev
 364         initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1]
 365         initialTag = "p4/%s" % (int(rev) - 1)
 366     except:
 367         pass
 368
 369 tz = - time.timezone / 36
 370 tzsign = ("%s" % tz)[0]
 371 if tzsign != '+' and tzsign != '-':
 372     tz = "+" + ("%s" % tz)
 373
 374 gitOutput, gitStream, gitError = popen2.popen3("git-fast-import")
 375
 376 if len(revision) > 0:
 377     print "Doing initial import of %s from revision %s" % (globalPrefix, revision)
 378
 379     details = { "user" : "git perforce import user", "time" : int(time.time()) }
 380     details["desc"] = "Initial import of %s from the state at revision %s" % (globalPrefix, revision)
 381     details["change"] = revision
 382     newestRevision = 0
 383
 384     fileCnt = 0
 385     for info in p4CmdList("files %s...%s" % (globalPrefix, revision)):
 386         change = int(info["change"])
 387         if change > newestRevision:
 388             newestRevision = change
 389
 390         if info["action"] == "delete":
 391             continue
 392
 393         for prop in [ "depotFile", "rev", "action", "type" ]:
 394             details["%s%s" % (prop, fileCnt)] = info[prop]
 395
 396         fileCnt = fileCnt + 1
 397
 398     details["change"] = newestRevision
 399
 400     try:
 401         commit(details, extractFilesFromCommit(details), branch, globalPrefix)
 402     except:
 403         print gitError.read()
 404
 405 else:
 406     changes = []
 407
 408     if len(changesFile) > 0:
 409         output = open(changesFile).readlines()
 410         changeSet = Set()
 411         for line in output:
 412             changeSet.add(int(line))
 413
 414         for change in changeSet:
 415             changes.append(change)
 416
 417         changes.sort()
 418     else:
 419         output = os.popen("p4 changes %s...%s" % (globalPrefix, changeRange)).readlines()
 420
 421         for line in output:
 422             changeNum = line.split(" ")[1]
 423             changes.append(changeNum)
 424
 425         changes.reverse()
 426
 427     if len(changes) == 0:
 428         if not silent:
 429             print "no changes to import!"
 430         sys.exit(1)
 431
 432     cnt = 1
 433     for change in changes:
 434         description = p4Cmd("describe %s" % change)
 435
 436         if not silent:
 437             sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
 438             sys.stdout.flush()
 439         cnt = cnt + 1
 440
 441 #        try:
 442         files = extractFilesFromCommit(description)
 443         if detectBranches:
 444             for branch in branchesForCommit(files):
 445                 knownBranches.add(branch)
 446                 branchPrefix = globalPrefix + branch + "/"
 447
 448                 parent = ""
 449                 ########### remove cnt!!!
 450                 if branch not in createdBranches and cnt > 2:
 451                     createdBranches.add(branch)
 452                     parent = findBranchParent(branchPrefix, files)
 453                     if parent == branch:
 454                         parent = ""
 455 #                    elif len(parent) > 0:
 456 #                        print "%s branched off of %s" % (branch, parent)
 457
 458                 branch = "refs/heads/" + branch
 459                 commit(description, files, branch, branchPrefix, parent)
 460         else:
 461             commit(description, files, branch, globalPrefix, initialParent)
 462             initialParent = ""
 463 #        except:
 464 #            print gitError.read()
 465 #            sys.exit(1)
 466
 467 if not silent:
 468     print ""
 469
 470 gitStream.write("reset refs/tags/p4/%s\n" % lastChange)
 471 gitStream.write("from %s\n\n" % branch);
 472
 473
 474 gitStream.close()
 475 gitOutput.close()
 476 gitError.close()
 477
 478 os.popen("git-repo-config p4.depotpath %s" % globalPrefix).read()
 479 if len(initialTag) > 0:
 480     os.popen("git tag -d %s" % initialTag).read()
 481
 482 sys.exit(0)