p4-fast-export.py

   1 #!/usr/bin/python
   2 #
   3 # p4-fast-export.py
   4 #
   5 # Author: Simon Hausmann <hausmann@kde.org>
   6 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   7 #
   8 # TODO:
   9 #       - support integrations (at least p4i)
  10 #       - support p4 submit (hah!)
  11 #
  12 import os, string, sys, time
  13 import marshal, popen2, getopt
  14 from sets import Set;
  15
  16 knownBranches = Set()
  17 committedChanges = Set()
  18 branch = "refs/heads/master"
  19 globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read()
  20 detectBranches = False
  21 changesFile = ""
  22 if len(globalPrefix) != 0:
  23     globalPrefix = globalPrefix[:-1]
  24
  25 try:
  26     opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=" ])
  27 except getopt.GetoptError:
  28     print "fixme, syntax error"
  29     sys.exit(1)
  30
  31 for o, a in opts:
  32     if o == "--branch":
  33         branch = "refs/heads/" + a
  34     elif o == "--detect-branches":
  35         detectBranches = True
  36     elif o == "--changesfile":
  37         changesFile = a
  38
  39 if len(args) == 0 and len(globalPrefix) != 0:
  40     print "[using previously specified depot path %s]" % globalPrefix
  41 elif len(args) != 1:
  42     print "usage: %s //depot/path[@revRange]" % sys.argv[0]
  43     print "\n    example:"
  44     print "    %s //depot/my/project/ -- to import the current head"
  45     print "    %s //depot/my/project/@all -- to import everything"
  46     print "    %s //depot/my/project/@1,6 -- to import only from revision 1 to 6"
  47     print ""
  48     print "    (a ... is not needed in the path p4 specification, it's added implicitly)"
  49     print ""
  50     sys.exit(1)
  51 else:
  52     if len(globalPrefix) != 0 and globalPrefix != args[0]:
  53         print "previous import used depot path %s and now %s was specified. this doesn't work!" % (globalPrefix, args[0])
  54         sys.exit(1)
  55     globalPrefix = args[0]
  56
  57 changeRange = ""
  58 revision = ""
  59 users = {}
  60 initialParent = ""
  61 lastChange = 0
  62 initialTag = ""
  63
  64 if globalPrefix.find("@") != -1:
  65     atIdx = globalPrefix.index("@")
  66     changeRange = globalPrefix[atIdx:]
  67     if changeRange == "@all":
  68         changeRange = ""
  69     elif changeRange.find(",") == -1:
  70         revision = changeRange
  71         changeRange = ""
  72     globalPrefix = globalPrefix[0:atIdx]
  73 elif globalPrefix.find("#") != -1:
  74     hashIdx = globalPrefix.index("#")
  75     revision = globalPrefix[hashIdx:]
  76     globalPrefix = globalPrefix[0:hashIdx]
  77 elif len(previousDepotPath) == 0:
  78     revision = "#head"
  79
  80 if globalPrefix.endswith("..."):
  81     globalPrefix = globalPrefix[:-3]
  82
  83 if not globalPrefix.endswith("/"):
  84     globalPrefix += "/"
  85
  86 def p4CmdList(cmd):
  87     pipe = os.popen("p4 -G %s" % cmd, "rb")
  88     result = []
  89     try:
  90         while True:
  91             entry = marshal.load(pipe)
  92             result.append(entry)
  93     except EOFError:
  94         pass
  95     pipe.close()
  96     return result
  97
  98 def p4Cmd(cmd):
  99     list = p4CmdList(cmd)
 100     result = {}
 101     for entry in list:
 102         result.update(entry)
 103     return result;
 104
 105 def extractFilesFromCommit(commit):
 106     files = []
 107     fnum = 0
 108     while commit.has_key("depotFile%s" % fnum):
 109         path =  commit["depotFile%s" % fnum]
 110         if not path.startswith(globalPrefix):
 111             print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change)
 112             fnum = fnum + 1
 113             continue
 114
 115         file = {}
 116         file["path"] = path
 117         file["rev"] = commit["rev%s" % fnum]
 118         file["action"] = commit["action%s" % fnum]
 119         file["type"] = commit["type%s" % fnum]
 120         files.append(file)
 121         fnum = fnum + 1
 122     return files
 123
 124 def isSubPathOf(first, second):
 125     if not first.startswith(second):
 126         return False
 127     if first == second:
 128         return True
 129     return first[len(second)] == "/"
 130
 131 def branchesForCommit(files):
 132     global knownBranches
 133     branches = Set()
 134
 135     for file in files:
 136         relativePath = file["path"][len(globalPrefix):]
 137         # strip off the filename
 138         relativePath = relativePath[0:relativePath.rfind("/")]
 139
 140 #        if len(branches) == 0:
 141 #            branches.add(relativePath)
 142 #            knownBranches.add(relativePath)
 143 #            continue
 144
 145         ###### this needs more testing :)
 146         knownBranch = False
 147         for branch in branches:
 148             if relativePath == branch:
 149                 knownBranch = True
 150                 break
 151 #            if relativePath.startswith(branch):
 152             if isSubPathOf(relativePath, branch):
 153                 knownBranch = True
 154                 break
 155 #            if branch.startswith(relativePath):
 156             if isSubPathOf(branch, relativePath):
 157                 branches.remove(branch)
 158                 break
 159
 160         if knownBranch:
 161             continue
 162
 163         for branch in knownBranches:
 164             #if relativePath.startswith(branch):
 165             if isSubPathOf(relativePath, branch):
 166                 if len(branches) == 0:
 167                     relativePath = branch
 168                 else:
 169                     knownBranch = True
 170                 break
 171
 172         if knownBranch:
 173             continue
 174
 175         branches.add(relativePath)
 176         knownBranches.add(relativePath)
 177
 178     return branches
 179
 180 def commit(details, files, branch, branchPrefix):
 181     global initialParent
 182     global users
 183     global lastChange
 184     global committedChanges
 185
 186     epoch = details["time"]
 187     author = details["user"]
 188
 189     gitStream.write("commit %s\n" % branch)
 190     gitStream.write("mark :%s\n" % details["change"])
 191     committedChanges.add(int(details["change"]))
 192     committer = ""
 193     if author in users:
 194         committer = "%s %s %s" % (users[author], epoch, tz)
 195     else:
 196         committer = "%s <a@b> %s %s" % (author, epoch, tz)
 197
 198     gitStream.write("committer %s\n" % committer)
 199
 200     gitStream.write("data <<EOT\n")
 201     gitStream.write(details["desc"])
 202     gitStream.write("\n[ imported from %s; change %s ]\n" % (branchPrefix, details["change"]))
 203     gitStream.write("EOT\n\n")
 204
 205     if len(initialParent) > 0:
 206         gitStream.write("from %s\n" % initialParent)
 207         initialParent = ""
 208
 209     #mergedBranches = Set()
 210     merges = Set()
 211
 212     for file in files:
 213         if lastChange == 0:
 214             continue
 215         path = file["path"]
 216         if not path.startswith(branchPrefix):
 217             continue
 218         action = file["action"]
 219         if action != "integrate" and action != "branch":
 220             continue
 221         rev = file["rev"]
 222         depotPath = path + "#" + rev
 223
 224         log = p4CmdList("filelog \"%s\"" % depotPath)
 225         if len(log) != 1:
 226             print "eek! I got confused by the filelog of %s" % depotPath
 227             sys.exit(1);
 228
 229         log = log[0]
 230         if log["action0"] != action:
 231             print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 232             sys.exit(1);
 233
 234         branchAction = log["how0,0"]
 235 #        if branchAction == "branch into" or branchAction == "ignored":
 236 #            continue # ignore for branching
 237
 238         if not branchAction.endswith(" from"):
 239             continue # ignore for branching
 240 #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 241 #            sys.exit(1);
 242
 243         source = log["file0,0"]
 244         if source.startswith(branchPrefix):
 245             continue
 246
 247         lastSourceRev = log["erev0,0"]
 248
 249         sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 250         if len(sourceLog) != 1:
 251             print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 252             sys.exit(1);
 253         sourceLog = sourceLog[0]
 254
 255         change = int(sourceLog["change0"])
 256         merges.add(change)
 257
 258 #        relPath = source[len(globalPrefix):]
 259 #
 260 #        for branch in knownBranches:
 261 #            if relPath.startswith(branch) and branch not in mergedBranches:
 262 #                gitStream.write("merge refs/heads/%s\n" % branch)
 263 #                mergedBranches.add(branch)
 264 #                break
 265
 266     for merge in merges:
 267         if merge in committedChanges:
 268             gitStream.write("merge :%s\n" % merge)
 269
 270     for file in files:
 271         path = file["path"]
 272         if not path.startswith(branchPrefix):
 273             print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 274             continue
 275         rev = file["rev"]
 276         depotPath = path + "#" + rev
 277         relPath = path[len(branchPrefix):]
 278         action = file["action"]
 279
 280         if action == "delete":
 281             gitStream.write("D %s\n" % relPath)
 282         else:
 283             mode = 644
 284             if file["type"].startswith("x"):
 285                 mode = 755
 286
 287             data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 288
 289             gitStream.write("M %s inline %s\n" % (mode, relPath))
 290             gitStream.write("data %s\n" % len(data))
 291             gitStream.write(data)
 292             gitStream.write("\n")
 293
 294     gitStream.write("\n")
 295
 296     lastChange = int(details["change"])
 297
 298 def getUserMap():
 299     users = {}
 300
 301     for output in p4CmdList("users"):
 302         if not output.has_key("User"):
 303             continue
 304         users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 305     return users
 306
 307 users = getUserMap()
 308
 309 if len(changeRange) == 0:
 310     try:
 311         sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
 312         output = sout.read()
 313         if output.endswith("\n"):
 314             output = output[:-1]
 315         tagIdx = output.index(" tags/p4/")
 316         caretIdx = output.find("^")
 317         endPos = len(output)
 318         if caretIdx != -1:
 319             endPos = caretIdx
 320         rev = int(output[tagIdx + 9 : endPos]) + 1
 321         changeRange = "@%s,#head" % rev
 322         initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1]
 323         initialTag = "p4/%s" % (int(rev) - 1)
 324     except:
 325         pass
 326
 327 sys.stderr.write("\n")
 328
 329 tz = - time.timezone / 36
 330 tzsign = ("%s" % tz)[0]
 331 if tzsign != '+' and tzsign != '-':
 332     tz = "+" + ("%s" % tz)
 333
 334 gitOutput, gitStream, gitError = popen2.popen3("git-fast-import")
 335
 336 if len(revision) > 0:
 337     print "Doing initial import of %s from revision %s" % (globalPrefix, revision)
 338
 339     details = { "user" : "git perforce import user", "time" : int(time.time()) }
 340     details["desc"] = "Initial import of %s from the state at revision %s" % (globalPrefix, revision)
 341     details["change"] = revision
 342     newestRevision = 0
 343
 344     fileCnt = 0
 345     for info in p4CmdList("files %s...%s" % (globalPrefix, revision)):
 346         change = int(info["change"])
 347         if change > newestRevision:
 348             newestRevision = change
 349
 350         if info["action"] == "delete":
 351             continue
 352
 353         for prop in [ "depotFile", "rev", "action", "type" ]:
 354             details["%s%s" % (prop, fileCnt)] = info[prop]
 355
 356         fileCnt = fileCnt + 1
 357
 358     details["change"] = newestRevision
 359
 360     try:
 361         commit(details, extractFilesFromCommit(details), branch, globalPrefix)
 362     except:
 363         print gitError.read()
 364
 365 else:
 366     changes = []
 367
 368     if len(changesFile) > 0:
 369         output = open(changesFile).readlines()
 370         changeSet = Set()
 371         for line in output:
 372             changeSet.add(int(line))
 373
 374         for change in changeSet:
 375             changes.append(change)
 376
 377         changes.sort()
 378     else:
 379         output = os.popen("p4 changes %s...%s" % (globalPrefix, changeRange)).readlines()
 380
 381         for line in output:
 382             changeNum = line.split(" ")[1]
 383             changes.append(changeNum)
 384
 385         changes.reverse()
 386
 387     if len(changes) == 0:
 388         print "no changes to import!"
 389         sys.exit(1)
 390
 391     cnt = 1
 392     for change in changes:
 393         description = p4Cmd("describe %s" % change)
 394
 395         sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
 396         sys.stdout.flush()
 397         cnt = cnt + 1
 398
 399 #        try:
 400         files = extractFilesFromCommit(description)
 401         if detectBranches:
 402             for branch in branchesForCommit(files):
 403                 knownBranches.add(branch)
 404                 branchPrefix = globalPrefix + branch + "/"
 405                 branch = "refs/heads/" + branch
 406                 commit(description, files, branch, branchPrefix)
 407         else:
 408             commit(description, files, branch, globalPrefix)
 409 #        except:
 410 #            print gitError.read()
 411 #            sys.exit(1)
 412
 413 print ""
 414
 415 gitStream.write("reset refs/tags/p4/%s\n" % lastChange)
 416 gitStream.write("from %s\n\n" % branch);
 417
 418
 419 gitStream.close()
 420 gitOutput.close()
 421 gitError.close()
 422
 423 os.popen("git-repo-config p4.depotpath %s" % globalPrefix).read()
 424 if len(initialTag) > 0:
 425     os.popen("git tag -d %s" % initialTag).read()
 426
 427 sys.exit(0)