sparse: Fix an "symbol 'cmd_index_pack' not declared" warning
[git.git] / contrib / fast-import / git-p4
blob78e5b3aaf4b70b87afbd5df0c9f839776310d18a
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse, sys, os, marshal, subprocess, shelve
12 import tempfile, getopt, os.path, time, platform
13 import re
15 verbose = False
18 def p4_build_cmd(cmd):
19 """Build a suitable p4 command line.
21 This consolidates building and returning a p4 command line into one
22 location. It means that hooking into the environment, or other configuration
23 can be done more easily.
24 """
25 real_cmd = "%s " % "p4"
27 user = gitConfig("git-p4.user")
28 if len(user) > 0:
29 real_cmd += "-u %s " % user
31 password = gitConfig("git-p4.password")
32 if len(password) > 0:
33 real_cmd += "-P %s " % password
35 port = gitConfig("git-p4.port")
36 if len(port) > 0:
37 real_cmd += "-p %s " % port
39 host = gitConfig("git-p4.host")
40 if len(host) > 0:
41 real_cmd += "-h %s " % host
43 client = gitConfig("git-p4.client")
44 if len(client) > 0:
45 real_cmd += "-c %s " % client
47 real_cmd += "%s" % (cmd)
48 if verbose:
49 print real_cmd
50 return real_cmd
52 def chdir(dir):
53 if os.name == 'nt':
54 os.environ['PWD']=dir
55 os.chdir(dir)
57 def die(msg):
58 if verbose:
59 raise Exception(msg)
60 else:
61 sys.stderr.write(msg + "\n")
62 sys.exit(1)
64 def write_pipe(c, str):
65 if verbose:
66 sys.stderr.write('Writing pipe: %s\n' % c)
68 pipe = os.popen(c, 'w')
69 val = pipe.write(str)
70 if pipe.close():
71 die('Command failed: %s' % c)
73 return val
75 def p4_write_pipe(c, str):
76 real_cmd = p4_build_cmd(c)
77 return write_pipe(real_cmd, str)
79 def read_pipe(c, ignore_error=False):
80 if verbose:
81 sys.stderr.write('Reading pipe: %s\n' % c)
83 pipe = os.popen(c, 'rb')
84 val = pipe.read()
85 if pipe.close() and not ignore_error:
86 die('Command failed: %s' % c)
88 return val
90 def p4_read_pipe(c, ignore_error=False):
91 real_cmd = p4_build_cmd(c)
92 return read_pipe(real_cmd, ignore_error)
94 def read_pipe_lines(c):
95 if verbose:
96 sys.stderr.write('Reading pipe: %s\n' % c)
97 ## todo: check return status
98 pipe = os.popen(c, 'rb')
99 val = pipe.readlines()
100 if pipe.close():
101 die('Command failed: %s' % c)
103 return val
105 def p4_read_pipe_lines(c):
106 """Specifically invoke p4 on the command supplied. """
107 real_cmd = p4_build_cmd(c)
108 return read_pipe_lines(real_cmd)
110 def system(cmd):
111 if verbose:
112 sys.stderr.write("executing %s\n" % cmd)
113 if os.system(cmd) != 0:
114 die("command failed: %s" % cmd)
116 def p4_system(cmd):
117 """Specifically invoke p4 as the system command. """
118 real_cmd = p4_build_cmd(cmd)
119 return system(real_cmd)
121 def isP4Exec(kind):
122 """Determine if a Perforce 'kind' should have execute permission
124 'p4 help filetypes' gives a list of the types. If it starts with 'x',
125 or x follows one of a few letters. Otherwise, if there is an 'x' after
126 a plus sign, it is also executable"""
127 return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
129 def setP4ExecBit(file, mode):
130 # Reopens an already open file and changes the execute bit to match
131 # the execute bit setting in the passed in mode.
133 p4Type = "+x"
135 if not isModeExec(mode):
136 p4Type = getP4OpenedType(file)
137 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
138 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
139 if p4Type[-1] == "+":
140 p4Type = p4Type[0:-1]
142 p4_system("reopen -t %s %s" % (p4Type, file))
144 def getP4OpenedType(file):
145 # Returns the perforce file type for the given file.
147 result = p4_read_pipe("opened %s" % file)
148 match = re.match(".*\((.+)\)\r?$", result)
149 if match:
150 return match.group(1)
151 else:
152 die("Could not determine file type for %s (result: '%s')" % (file, result))
154 def diffTreePattern():
155 # This is a simple generator for the diff tree regex pattern. This could be
156 # a class variable if this and parseDiffTreeEntry were a part of a class.
157 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
158 while True:
159 yield pattern
161 def parseDiffTreeEntry(entry):
162 """Parses a single diff tree entry into its component elements.
164 See git-diff-tree(1) manpage for details about the format of the diff
165 output. This method returns a dictionary with the following elements:
167 src_mode - The mode of the source file
168 dst_mode - The mode of the destination file
169 src_sha1 - The sha1 for the source file
170 dst_sha1 - The sha1 fr the destination file
171 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
172 status_score - The score for the status (applicable for 'C' and 'R'
173 statuses). This is None if there is no score.
174 src - The path for the source file.
175 dst - The path for the destination file. This is only present for
176 copy or renames. If it is not present, this is None.
178 If the pattern is not matched, None is returned."""
180 match = diffTreePattern().next().match(entry)
181 if match:
182 return {
183 'src_mode': match.group(1),
184 'dst_mode': match.group(2),
185 'src_sha1': match.group(3),
186 'dst_sha1': match.group(4),
187 'status': match.group(5),
188 'status_score': match.group(6),
189 'src': match.group(7),
190 'dst': match.group(10)
192 return None
194 def isModeExec(mode):
195 # Returns True if the given git mode represents an executable file,
196 # otherwise False.
197 return mode[-3:] == "755"
199 def isModeExecChanged(src_mode, dst_mode):
200 return isModeExec(src_mode) != isModeExec(dst_mode)
202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
203 cmd = p4_build_cmd("-G %s" % (cmd))
204 if verbose:
205 sys.stderr.write("Opening pipe: %s\n" % cmd)
207 # Use a temporary file to avoid deadlocks without
208 # subprocess.communicate(), which would put another copy
209 # of stdout into memory.
210 stdin_file = None
211 if stdin is not None:
212 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
213 stdin_file.write(stdin)
214 stdin_file.flush()
215 stdin_file.seek(0)
217 p4 = subprocess.Popen(cmd, shell=True,
218 stdin=stdin_file,
219 stdout=subprocess.PIPE)
221 result = []
222 try:
223 while True:
224 entry = marshal.load(p4.stdout)
225 if cb is not None:
226 cb(entry)
227 else:
228 result.append(entry)
229 except EOFError:
230 pass
231 exitCode = p4.wait()
232 if exitCode != 0:
233 entry = {}
234 entry["p4ExitCode"] = exitCode
235 result.append(entry)
237 return result
239 def p4Cmd(cmd):
240 list = p4CmdList(cmd)
241 result = {}
242 for entry in list:
243 result.update(entry)
244 return result;
246 def p4Where(depotPath):
247 if not depotPath.endswith("/"):
248 depotPath += "/"
249 depotPath = depotPath + "..."
250 outputList = p4CmdList("where %s" % depotPath)
251 output = None
252 for entry in outputList:
253 if "depotFile" in entry:
254 if entry["depotFile"] == depotPath:
255 output = entry
256 break
257 elif "data" in entry:
258 data = entry.get("data")
259 space = data.find(" ")
260 if data[:space] == depotPath:
261 output = entry
262 break
263 if output == None:
264 return ""
265 if output["code"] == "error":
266 return ""
267 clientPath = ""
268 if "path" in output:
269 clientPath = output.get("path")
270 elif "data" in output:
271 data = output.get("data")
272 lastSpace = data.rfind(" ")
273 clientPath = data[lastSpace + 1:]
275 if clientPath.endswith("..."):
276 clientPath = clientPath[:-3]
277 return clientPath
279 def currentGitBranch():
280 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
282 def isValidGitDir(path):
283 if (os.path.exists(path + "/HEAD")
284 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
285 return True;
286 return False
288 def parseRevision(ref):
289 return read_pipe("git rev-parse %s" % ref).strip()
291 def extractLogMessageFromGitCommit(commit):
292 logMessage = ""
294 ## fixme: title is first line of commit, not 1st paragraph.
295 foundTitle = False
296 for log in read_pipe_lines("git cat-file commit %s" % commit):
297 if not foundTitle:
298 if len(log) == 1:
299 foundTitle = True
300 continue
302 logMessage += log
303 return logMessage
305 def extractSettingsGitLog(log):
306 values = {}
307 for line in log.split("\n"):
308 line = line.strip()
309 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
310 if not m:
311 continue
313 assignments = m.group(1).split (':')
314 for a in assignments:
315 vals = a.split ('=')
316 key = vals[0].strip()
317 val = ('='.join (vals[1:])).strip()
318 if val.endswith ('\"') and val.startswith('"'):
319 val = val[1:-1]
321 values[key] = val
323 paths = values.get("depot-paths")
324 if not paths:
325 paths = values.get("depot-path")
326 if paths:
327 values['depot-paths'] = paths.split(',')
328 return values
330 def gitBranchExists(branch):
331 proc = subprocess.Popen(["git", "rev-parse", branch],
332 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
333 return proc.wait() == 0;
335 _gitConfig = {}
336 def gitConfig(key, args = None): # set args to "--bool", for instance
337 if not _gitConfig.has_key(key):
338 argsFilter = ""
339 if args != None:
340 argsFilter = "%s " % args
341 cmd = "git config %s%s" % (argsFilter, key)
342 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
343 return _gitConfig[key]
345 def p4BranchesInGit(branchesAreInRemotes = True):
346 branches = {}
348 cmdline = "git rev-parse --symbolic "
349 if branchesAreInRemotes:
350 cmdline += " --remotes"
351 else:
352 cmdline += " --branches"
354 for line in read_pipe_lines(cmdline):
355 line = line.strip()
357 ## only import to p4/
358 if not line.startswith('p4/') or line == "p4/HEAD":
359 continue
360 branch = line
362 # strip off p4
363 branch = re.sub ("^p4/", "", line)
365 branches[branch] = parseRevision(line)
366 return branches
368 def findUpstreamBranchPoint(head = "HEAD"):
369 branches = p4BranchesInGit()
370 # map from depot-path to branch name
371 branchByDepotPath = {}
372 for branch in branches.keys():
373 tip = branches[branch]
374 log = extractLogMessageFromGitCommit(tip)
375 settings = extractSettingsGitLog(log)
376 if settings.has_key("depot-paths"):
377 paths = ",".join(settings["depot-paths"])
378 branchByDepotPath[paths] = "remotes/p4/" + branch
380 settings = None
381 parent = 0
382 while parent < 65535:
383 commit = head + "~%s" % parent
384 log = extractLogMessageFromGitCommit(commit)
385 settings = extractSettingsGitLog(log)
386 if settings.has_key("depot-paths"):
387 paths = ",".join(settings["depot-paths"])
388 if branchByDepotPath.has_key(paths):
389 return [branchByDepotPath[paths], settings]
391 parent = parent + 1
393 return ["", settings]
395 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
396 if not silent:
397 print ("Creating/updating branch(es) in %s based on origin branch(es)"
398 % localRefPrefix)
400 originPrefix = "origin/p4/"
402 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
403 line = line.strip()
404 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
405 continue
407 headName = line[len(originPrefix):]
408 remoteHead = localRefPrefix + headName
409 originHead = line
411 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
412 if (not original.has_key('depot-paths')
413 or not original.has_key('change')):
414 continue
416 update = False
417 if not gitBranchExists(remoteHead):
418 if verbose:
419 print "creating %s" % remoteHead
420 update = True
421 else:
422 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
423 if settings.has_key('change') > 0:
424 if settings['depot-paths'] == original['depot-paths']:
425 originP4Change = int(original['change'])
426 p4Change = int(settings['change'])
427 if originP4Change > p4Change:
428 print ("%s (%s) is newer than %s (%s). "
429 "Updating p4 branch from origin."
430 % (originHead, originP4Change,
431 remoteHead, p4Change))
432 update = True
433 else:
434 print ("Ignoring: %s was imported from %s while "
435 "%s was imported from %s"
436 % (originHead, ','.join(original['depot-paths']),
437 remoteHead, ','.join(settings['depot-paths'])))
439 if update:
440 system("git update-ref %s %s" % (remoteHead, originHead))
442 def originP4BranchesExist():
443 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
445 def p4ChangesForPaths(depotPaths, changeRange):
446 assert depotPaths
447 output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
448 for p in depotPaths]))
450 changes = {}
451 for line in output:
452 changeNum = int(line.split(" ")[1])
453 changes[changeNum] = True
455 changelist = changes.keys()
456 changelist.sort()
457 return changelist
459 def p4PathStartsWith(path, prefix):
460 # This method tries to remedy a potential mixed-case issue:
462 # If UserA adds //depot/DirA/file1
463 # and UserB adds //depot/dira/file2
465 # we may or may not have a problem. If you have core.ignorecase=true,
466 # we treat DirA and dira as the same directory
467 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
468 if ignorecase:
469 return path.lower().startswith(prefix.lower())
470 return path.startswith(prefix)
472 class Command:
473 def __init__(self):
474 self.usage = "usage: %prog [options]"
475 self.needsGit = True
477 class P4Debug(Command):
478 def __init__(self):
479 Command.__init__(self)
480 self.options = [
481 optparse.make_option("--verbose", dest="verbose", action="store_true",
482 default=False),
484 self.description = "A tool to debug the output of p4 -G."
485 self.needsGit = False
486 self.verbose = False
488 def run(self, args):
489 j = 0
490 for output in p4CmdList(" ".join(args)):
491 print 'Element: %d' % j
492 j += 1
493 print output
494 return True
496 class P4RollBack(Command):
497 def __init__(self):
498 Command.__init__(self)
499 self.options = [
500 optparse.make_option("--verbose", dest="verbose", action="store_true"),
501 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
503 self.description = "A tool to debug the multi-branch import. Don't use :)"
504 self.verbose = False
505 self.rollbackLocalBranches = False
507 def run(self, args):
508 if len(args) != 1:
509 return False
510 maxChange = int(args[0])
512 if "p4ExitCode" in p4Cmd("changes -m 1"):
513 die("Problems executing p4");
515 if self.rollbackLocalBranches:
516 refPrefix = "refs/heads/"
517 lines = read_pipe_lines("git rev-parse --symbolic --branches")
518 else:
519 refPrefix = "refs/remotes/"
520 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
522 for line in lines:
523 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
524 line = line.strip()
525 ref = refPrefix + line
526 log = extractLogMessageFromGitCommit(ref)
527 settings = extractSettingsGitLog(log)
529 depotPaths = settings['depot-paths']
530 change = settings['change']
532 changed = False
534 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
535 for p in depotPaths]))) == 0:
536 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
537 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
538 continue
540 while change and int(change) > maxChange:
541 changed = True
542 if self.verbose:
543 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
544 system("git update-ref %s \"%s^\"" % (ref, ref))
545 log = extractLogMessageFromGitCommit(ref)
546 settings = extractSettingsGitLog(log)
549 depotPaths = settings['depot-paths']
550 change = settings['change']
552 if changed:
553 print "%s rewound to %s" % (ref, change)
555 return True
557 class P4Submit(Command):
558 def __init__(self):
559 Command.__init__(self)
560 self.options = [
561 optparse.make_option("--verbose", dest="verbose", action="store_true"),
562 optparse.make_option("--origin", dest="origin"),
563 optparse.make_option("-M", dest="detectRenames", action="store_true"),
565 self.description = "Submit changes from git to the perforce depot."
566 self.usage += " [name of git branch to submit into perforce depot]"
567 self.interactive = True
568 self.origin = ""
569 self.detectRenames = False
570 self.verbose = False
571 self.isWindows = (platform.system() == "Windows")
573 def check(self):
574 if len(p4CmdList("opened ...")) > 0:
575 die("You have files opened with perforce! Close them before starting the sync.")
577 # replaces everything between 'Description:' and the next P4 submit template field with the
578 # commit message
579 def prepareLogMessage(self, template, message):
580 result = ""
582 inDescriptionSection = False
584 for line in template.split("\n"):
585 if line.startswith("#"):
586 result += line + "\n"
587 continue
589 if inDescriptionSection:
590 if line.startswith("Files:") or line.startswith("Jobs:"):
591 inDescriptionSection = False
592 else:
593 continue
594 else:
595 if line.startswith("Description:"):
596 inDescriptionSection = True
597 line += "\n"
598 for messageLine in message.split("\n"):
599 line += "\t" + messageLine + "\n"
601 result += line + "\n"
603 return result
605 def prepareSubmitTemplate(self):
606 # remove lines in the Files section that show changes to files outside the depot path we're committing into
607 template = ""
608 inFilesSection = False
609 for line in p4_read_pipe_lines("change -o"):
610 if line.endswith("\r\n"):
611 line = line[:-2] + "\n"
612 if inFilesSection:
613 if line.startswith("\t"):
614 # path starts and ends with a tab
615 path = line[1:]
616 lastTab = path.rfind("\t")
617 if lastTab != -1:
618 path = path[:lastTab]
619 if not p4PathStartsWith(path, self.depotPath):
620 continue
621 else:
622 inFilesSection = False
623 else:
624 if line.startswith("Files:"):
625 inFilesSection = True
627 template += line
629 return template
631 def applyCommit(self, id):
632 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
634 if not self.detectRenames:
635 # If not explicitly set check the config variable
636 self.detectRenames = gitConfig("git-p4.detectRenames").lower() == "true"
638 if self.detectRenames:
639 diffOpts = "-M"
640 else:
641 diffOpts = ""
643 if gitConfig("git-p4.detectCopies").lower() == "true":
644 diffOpts += " -C"
646 if gitConfig("git-p4.detectCopiesHarder").lower() == "true":
647 diffOpts += " --find-copies-harder"
649 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
650 filesToAdd = set()
651 filesToDelete = set()
652 editedFiles = set()
653 filesToChangeExecBit = {}
654 for line in diff:
655 diff = parseDiffTreeEntry(line)
656 modifier = diff['status']
657 path = diff['src']
658 if modifier == "M":
659 p4_system("edit \"%s\"" % path)
660 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
661 filesToChangeExecBit[path] = diff['dst_mode']
662 editedFiles.add(path)
663 elif modifier == "A":
664 filesToAdd.add(path)
665 filesToChangeExecBit[path] = diff['dst_mode']
666 if path in filesToDelete:
667 filesToDelete.remove(path)
668 elif modifier == "D":
669 filesToDelete.add(path)
670 if path in filesToAdd:
671 filesToAdd.remove(path)
672 elif modifier == "C":
673 src, dest = diff['src'], diff['dst']
674 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
675 if diff['src_sha1'] != diff['dst_sha1']:
676 p4_system("edit \"%s\"" % (dest))
677 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
678 p4_system("edit \"%s\"" % (dest))
679 filesToChangeExecBit[dest] = diff['dst_mode']
680 os.unlink(dest)
681 editedFiles.add(dest)
682 elif modifier == "R":
683 src, dest = diff['src'], diff['dst']
684 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
685 if diff['src_sha1'] != diff['dst_sha1']:
686 p4_system("edit \"%s\"" % (dest))
687 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
688 p4_system("edit \"%s\"" % (dest))
689 filesToChangeExecBit[dest] = diff['dst_mode']
690 os.unlink(dest)
691 editedFiles.add(dest)
692 filesToDelete.add(src)
693 else:
694 die("unknown modifier %s for %s" % (modifier, path))
696 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
697 patchcmd = diffcmd + " | git apply "
698 tryPatchCmd = patchcmd + "--check -"
699 applyPatchCmd = patchcmd + "--check --apply -"
701 if os.system(tryPatchCmd) != 0:
702 print "Unfortunately applying the change failed!"
703 print "What do you want to do?"
704 response = "x"
705 while response != "s" and response != "a" and response != "w":
706 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
707 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
708 if response == "s":
709 print "Skipping! Good luck with the next patches..."
710 for f in editedFiles:
711 p4_system("revert \"%s\"" % f);
712 for f in filesToAdd:
713 system("rm %s" %f)
714 return
715 elif response == "a":
716 os.system(applyPatchCmd)
717 if len(filesToAdd) > 0:
718 print "You may also want to call p4 add on the following files:"
719 print " ".join(filesToAdd)
720 if len(filesToDelete):
721 print "The following files should be scheduled for deletion with p4 delete:"
722 print " ".join(filesToDelete)
723 die("Please resolve and submit the conflict manually and "
724 + "continue afterwards with git-p4 submit --continue")
725 elif response == "w":
726 system(diffcmd + " > patch.txt")
727 print "Patch saved to patch.txt in %s !" % self.clientPath
728 die("Please resolve and submit the conflict manually and "
729 "continue afterwards with git-p4 submit --continue")
731 system(applyPatchCmd)
733 for f in filesToAdd:
734 p4_system("add \"%s\"" % f)
735 for f in filesToDelete:
736 p4_system("revert \"%s\"" % f)
737 p4_system("delete \"%s\"" % f)
739 # Set/clear executable bits
740 for f in filesToChangeExecBit.keys():
741 mode = filesToChangeExecBit[f]
742 setP4ExecBit(f, mode)
744 logMessage = extractLogMessageFromGitCommit(id)
745 logMessage = logMessage.strip()
747 template = self.prepareSubmitTemplate()
749 if self.interactive:
750 submitTemplate = self.prepareLogMessage(template, logMessage)
751 if os.environ.has_key("P4DIFF"):
752 del(os.environ["P4DIFF"])
753 diff = ""
754 for editedFile in editedFiles:
755 diff += p4_read_pipe("diff -du %r" % editedFile)
757 newdiff = ""
758 for newFile in filesToAdd:
759 newdiff += "==== new file ====\n"
760 newdiff += "--- /dev/null\n"
761 newdiff += "+++ %s\n" % newFile
762 f = open(newFile, "r")
763 for line in f.readlines():
764 newdiff += "+" + line
765 f.close()
767 separatorLine = "######## everything below this line is just the diff #######\n"
769 [handle, fileName] = tempfile.mkstemp()
770 tmpFile = os.fdopen(handle, "w+")
771 if self.isWindows:
772 submitTemplate = submitTemplate.replace("\n", "\r\n")
773 separatorLine = separatorLine.replace("\n", "\r\n")
774 newdiff = newdiff.replace("\n", "\r\n")
775 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
776 tmpFile.close()
777 mtime = os.stat(fileName).st_mtime
778 if os.environ.has_key("P4EDITOR"):
779 editor = os.environ.get("P4EDITOR")
780 else:
781 editor = read_pipe("git var GIT_EDITOR").strip()
782 system(editor + " " + fileName)
784 response = "y"
785 if os.stat(fileName).st_mtime <= mtime:
786 response = "x"
787 while response != "y" and response != "n":
788 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
790 if response == "y":
791 tmpFile = open(fileName, "rb")
792 message = tmpFile.read()
793 tmpFile.close()
794 submitTemplate = message[:message.index(separatorLine)]
795 if self.isWindows:
796 submitTemplate = submitTemplate.replace("\r\n", "\n")
797 p4_write_pipe("submit -i", submitTemplate)
798 else:
799 for f in editedFiles:
800 p4_system("revert \"%s\"" % f);
801 for f in filesToAdd:
802 p4_system("revert \"%s\"" % f);
803 system("rm %s" %f)
805 os.remove(fileName)
806 else:
807 fileName = "submit.txt"
808 file = open(fileName, "w+")
809 file.write(self.prepareLogMessage(template, logMessage))
810 file.close()
811 print ("Perforce submit template written as %s. "
812 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
813 % (fileName, fileName))
815 def run(self, args):
816 if len(args) == 0:
817 self.master = currentGitBranch()
818 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
819 die("Detecting current git branch failed!")
820 elif len(args) == 1:
821 self.master = args[0]
822 else:
823 return False
825 allowSubmit = gitConfig("git-p4.allowSubmit")
826 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
827 die("%s is not in git-p4.allowSubmit" % self.master)
829 [upstream, settings] = findUpstreamBranchPoint()
830 self.depotPath = settings['depot-paths'][0]
831 if len(self.origin) == 0:
832 self.origin = upstream
834 if self.verbose:
835 print "Origin branch is " + self.origin
837 if len(self.depotPath) == 0:
838 print "Internal error: cannot locate perforce depot path from existing branches"
839 sys.exit(128)
841 self.clientPath = p4Where(self.depotPath)
843 if len(self.clientPath) == 0:
844 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
845 sys.exit(128)
847 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
848 self.oldWorkingDirectory = os.getcwd()
850 chdir(self.clientPath)
851 print "Synchronizing p4 checkout..."
852 p4_system("sync ...")
854 self.check()
856 commits = []
857 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
858 commits.append(line.strip())
859 commits.reverse()
861 while len(commits) > 0:
862 commit = commits[0]
863 commits = commits[1:]
864 self.applyCommit(commit)
865 if not self.interactive:
866 break
868 if len(commits) == 0:
869 print "All changes applied!"
870 chdir(self.oldWorkingDirectory)
872 sync = P4Sync()
873 sync.run([])
875 rebase = P4Rebase()
876 rebase.rebase()
878 return True
880 class P4Sync(Command):
881 delete_actions = ( "delete", "move/delete", "purge" )
883 def __init__(self):
884 Command.__init__(self)
885 self.options = [
886 optparse.make_option("--branch", dest="branch"),
887 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
888 optparse.make_option("--changesfile", dest="changesFile"),
889 optparse.make_option("--silent", dest="silent", action="store_true"),
890 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
891 optparse.make_option("--verbose", dest="verbose", action="store_true"),
892 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
893 help="Import into refs/heads/ , not refs/remotes"),
894 optparse.make_option("--max-changes", dest="maxChanges"),
895 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
896 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
897 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
898 help="Only sync files that are included in the Perforce Client Spec")
900 self.description = """Imports from Perforce into a git repository.\n
901 example:
902 //depot/my/project/ -- to import the current head
903 //depot/my/project/@all -- to import everything
904 //depot/my/project/@1,6 -- to import only from revision 1 to 6
906 (a ... is not needed in the path p4 specification, it's added implicitly)"""
908 self.usage += " //depot/path[@revRange]"
909 self.silent = False
910 self.createdBranches = set()
911 self.committedChanges = set()
912 self.branch = ""
913 self.detectBranches = False
914 self.detectLabels = False
915 self.changesFile = ""
916 self.syncWithOrigin = True
917 self.verbose = False
918 self.importIntoRemotes = True
919 self.maxChanges = ""
920 self.isWindows = (platform.system() == "Windows")
921 self.keepRepoPath = False
922 self.depotPaths = None
923 self.p4BranchesInGit = []
924 self.cloneExclude = []
925 self.useClientSpec = False
926 self.clientSpecDirs = []
928 if gitConfig("git-p4.syncFromOrigin") == "false":
929 self.syncWithOrigin = False
932 # P4 wildcards are not allowed in filenames. P4 complains
933 # if you simply add them, but you can force it with "-f", in
934 # which case it translates them into %xx encoding internally.
935 # Search for and fix just these four characters. Do % last so
936 # that fixing it does not inadvertently create new %-escapes.
938 def wildcard_decode(self, path):
939 # Cannot have * in a filename in windows; untested as to
940 # what p4 would do in such a case.
941 if not self.isWindows:
942 path = path.replace("%2A", "*")
943 path = path.replace("%23", "#") \
944 .replace("%40", "@") \
945 .replace("%25", "%")
946 return path
948 def extractFilesFromCommit(self, commit):
949 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
950 for path in self.cloneExclude]
951 files = []
952 fnum = 0
953 while commit.has_key("depotFile%s" % fnum):
954 path = commit["depotFile%s" % fnum]
956 if [p for p in self.cloneExclude
957 if p4PathStartsWith(path, p)]:
958 found = False
959 else:
960 found = [p for p in self.depotPaths
961 if p4PathStartsWith(path, p)]
962 if not found:
963 fnum = fnum + 1
964 continue
966 file = {}
967 file["path"] = path
968 file["rev"] = commit["rev%s" % fnum]
969 file["action"] = commit["action%s" % fnum]
970 file["type"] = commit["type%s" % fnum]
971 files.append(file)
972 fnum = fnum + 1
973 return files
975 def stripRepoPath(self, path, prefixes):
976 if self.useClientSpec:
978 # if using the client spec, we use the output directory
979 # specified in the client. For example, a view
980 # //depot/foo/branch/... //client/branch/foo/...
981 # will end up putting all foo/branch files into
982 # branch/foo/
983 for val in self.clientSpecDirs:
984 if path.startswith(val[0]):
985 # replace the depot path with the client path
986 path = path.replace(val[0], val[1][1])
987 # now strip out the client (//client/...)
988 path = re.sub("^(//[^/]+/)", '', path)
989 # the rest is all path
990 return path
992 if self.keepRepoPath:
993 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
995 for p in prefixes:
996 if p4PathStartsWith(path, p):
997 path = path[len(p):]
999 return path
1001 def splitFilesIntoBranches(self, commit):
1002 branches = {}
1003 fnum = 0
1004 while commit.has_key("depotFile%s" % fnum):
1005 path = commit["depotFile%s" % fnum]
1006 found = [p for p in self.depotPaths
1007 if p4PathStartsWith(path, p)]
1008 if not found:
1009 fnum = fnum + 1
1010 continue
1012 file = {}
1013 file["path"] = path
1014 file["rev"] = commit["rev%s" % fnum]
1015 file["action"] = commit["action%s" % fnum]
1016 file["type"] = commit["type%s" % fnum]
1017 fnum = fnum + 1
1019 relPath = self.stripRepoPath(path, self.depotPaths)
1021 for branch in self.knownBranches.keys():
1023 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1024 if relPath.startswith(branch + "/"):
1025 if branch not in branches:
1026 branches[branch] = []
1027 branches[branch].append(file)
1028 break
1030 return branches
1032 # output one file from the P4 stream
1033 # - helper for streamP4Files
1035 def streamOneP4File(self, file, contents):
1036 if file["type"] == "apple":
1037 print "\nfile %s is a strange apple file that forks. Ignoring" % \
1038 file['depotFile']
1039 return
1041 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1042 relPath = self.wildcard_decode(relPath)
1043 if verbose:
1044 sys.stderr.write("%s\n" % relPath)
1046 mode = "644"
1047 if isP4Exec(file["type"]):
1048 mode = "755"
1049 elif file["type"] == "symlink":
1050 mode = "120000"
1051 # p4 print on a symlink contains "target\n", so strip it off
1052 data = ''.join(contents)
1053 contents = [data[:-1]]
1055 if self.isWindows and file["type"].endswith("text"):
1056 mangled = []
1057 for data in contents:
1058 data = data.replace("\r\n", "\n")
1059 mangled.append(data)
1060 contents = mangled
1062 if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1063 contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1064 elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1065 contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1067 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1069 # total length...
1070 length = 0
1071 for d in contents:
1072 length = length + len(d)
1074 self.gitStream.write("data %d\n" % length)
1075 for d in contents:
1076 self.gitStream.write(d)
1077 self.gitStream.write("\n")
1079 def streamOneP4Deletion(self, file):
1080 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1081 if verbose:
1082 sys.stderr.write("delete %s\n" % relPath)
1083 self.gitStream.write("D %s\n" % relPath)
1085 # handle another chunk of streaming data
1086 def streamP4FilesCb(self, marshalled):
1088 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1089 # start of a new file - output the old one first
1090 self.streamOneP4File(self.stream_file, self.stream_contents)
1091 self.stream_file = {}
1092 self.stream_contents = []
1093 self.stream_have_file_info = False
1095 # pick up the new file information... for the
1096 # 'data' field we need to append to our array
1097 for k in marshalled.keys():
1098 if k == 'data':
1099 self.stream_contents.append(marshalled['data'])
1100 else:
1101 self.stream_file[k] = marshalled[k]
1103 self.stream_have_file_info = True
1105 # Stream directly from "p4 files" into "git fast-import"
1106 def streamP4Files(self, files):
1107 filesForCommit = []
1108 filesToRead = []
1109 filesToDelete = []
1111 for f in files:
1112 includeFile = True
1113 for val in self.clientSpecDirs:
1114 if f['path'].startswith(val[0]):
1115 if val[1][0] <= 0:
1116 includeFile = False
1117 break
1119 if includeFile:
1120 filesForCommit.append(f)
1121 if f['action'] in self.delete_actions:
1122 filesToDelete.append(f)
1123 else:
1124 filesToRead.append(f)
1126 # deleted files...
1127 for f in filesToDelete:
1128 self.streamOneP4Deletion(f)
1130 if len(filesToRead) > 0:
1131 self.stream_file = {}
1132 self.stream_contents = []
1133 self.stream_have_file_info = False
1135 # curry self argument
1136 def streamP4FilesCbSelf(entry):
1137 self.streamP4FilesCb(entry)
1139 p4CmdList("-x - print",
1140 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1141 for f in filesToRead]),
1142 cb=streamP4FilesCbSelf)
1144 # do the last chunk
1145 if self.stream_file.has_key('depotFile'):
1146 self.streamOneP4File(self.stream_file, self.stream_contents)
1148 def commit(self, details, files, branch, branchPrefixes, parent = ""):
1149 epoch = details["time"]
1150 author = details["user"]
1151 self.branchPrefixes = branchPrefixes
1153 if self.verbose:
1154 print "commit into %s" % branch
1156 # start with reading files; if that fails, we should not
1157 # create a commit.
1158 new_files = []
1159 for f in files:
1160 if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1161 new_files.append (f)
1162 else:
1163 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1165 self.gitStream.write("commit %s\n" % branch)
1166 # gitStream.write("mark :%s\n" % details["change"])
1167 self.committedChanges.add(int(details["change"]))
1168 committer = ""
1169 if author not in self.users:
1170 self.getUserMapFromPerforceServer()
1171 if author in self.users:
1172 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1173 else:
1174 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1176 self.gitStream.write("committer %s\n" % committer)
1178 self.gitStream.write("data <<EOT\n")
1179 self.gitStream.write(details["desc"])
1180 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1181 % (','.join (branchPrefixes), details["change"]))
1182 if len(details['options']) > 0:
1183 self.gitStream.write(": options = %s" % details['options'])
1184 self.gitStream.write("]\nEOT\n\n")
1186 if len(parent) > 0:
1187 if self.verbose:
1188 print "parent %s" % parent
1189 self.gitStream.write("from %s\n" % parent)
1191 self.streamP4Files(new_files)
1192 self.gitStream.write("\n")
1194 change = int(details["change"])
1196 if self.labels.has_key(change):
1197 label = self.labels[change]
1198 labelDetails = label[0]
1199 labelRevisions = label[1]
1200 if self.verbose:
1201 print "Change %s is labelled %s" % (change, labelDetails)
1203 files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1204 for p in branchPrefixes]))
1206 if len(files) == len(labelRevisions):
1208 cleanedFiles = {}
1209 for info in files:
1210 if info["action"] in self.delete_actions:
1211 continue
1212 cleanedFiles[info["depotFile"]] = info["rev"]
1214 if cleanedFiles == labelRevisions:
1215 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1216 self.gitStream.write("from %s\n" % branch)
1218 owner = labelDetails["Owner"]
1219 tagger = ""
1220 if author in self.users:
1221 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1222 else:
1223 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1224 self.gitStream.write("tagger %s\n" % tagger)
1225 self.gitStream.write("data <<EOT\n")
1226 self.gitStream.write(labelDetails["Description"])
1227 self.gitStream.write("EOT\n\n")
1229 else:
1230 if not self.silent:
1231 print ("Tag %s does not match with change %s: files do not match."
1232 % (labelDetails["label"], change))
1234 else:
1235 if not self.silent:
1236 print ("Tag %s does not match with change %s: file count is different."
1237 % (labelDetails["label"], change))
1239 def getUserCacheFilename(self):
1240 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1241 return home + "/.gitp4-usercache.txt"
1243 def getUserMapFromPerforceServer(self):
1244 if self.userMapFromPerforceServer:
1245 return
1246 self.users = {}
1248 for output in p4CmdList("users"):
1249 if not output.has_key("User"):
1250 continue
1251 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1254 s = ''
1255 for (key, val) in self.users.items():
1256 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1258 open(self.getUserCacheFilename(), "wb").write(s)
1259 self.userMapFromPerforceServer = True
1261 def loadUserMapFromCache(self):
1262 self.users = {}
1263 self.userMapFromPerforceServer = False
1264 try:
1265 cache = open(self.getUserCacheFilename(), "rb")
1266 lines = cache.readlines()
1267 cache.close()
1268 for line in lines:
1269 entry = line.strip().split("\t")
1270 self.users[entry[0]] = entry[1]
1271 except IOError:
1272 self.getUserMapFromPerforceServer()
1274 def getLabels(self):
1275 self.labels = {}
1277 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1278 if len(l) > 0 and not self.silent:
1279 print "Finding files belonging to labels in %s" % `self.depotPaths`
1281 for output in l:
1282 label = output["label"]
1283 revisions = {}
1284 newestChange = 0
1285 if self.verbose:
1286 print "Querying files for label %s" % label
1287 for file in p4CmdList("files "
1288 + ' '.join (["%s...@%s" % (p, label)
1289 for p in self.depotPaths])):
1290 revisions[file["depotFile"]] = file["rev"]
1291 change = int(file["change"])
1292 if change > newestChange:
1293 newestChange = change
1295 self.labels[newestChange] = [output, revisions]
1297 if self.verbose:
1298 print "Label changes: %s" % self.labels.keys()
1300 def guessProjectName(self):
1301 for p in self.depotPaths:
1302 if p.endswith("/"):
1303 p = p[:-1]
1304 p = p[p.strip().rfind("/") + 1:]
1305 if not p.endswith("/"):
1306 p += "/"
1307 return p
1309 def getBranchMapping(self):
1310 lostAndFoundBranches = set()
1312 for info in p4CmdList("branches"):
1313 details = p4Cmd("branch -o %s" % info["branch"])
1314 viewIdx = 0
1315 while details.has_key("View%s" % viewIdx):
1316 paths = details["View%s" % viewIdx].split(" ")
1317 viewIdx = viewIdx + 1
1318 # require standard //depot/foo/... //depot/bar/... mapping
1319 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1320 continue
1321 source = paths[0]
1322 destination = paths[1]
1323 ## HACK
1324 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1325 source = source[len(self.depotPaths[0]):-4]
1326 destination = destination[len(self.depotPaths[0]):-4]
1328 if destination in self.knownBranches:
1329 if not self.silent:
1330 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1331 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1332 continue
1334 self.knownBranches[destination] = source
1336 lostAndFoundBranches.discard(destination)
1338 if source not in self.knownBranches:
1339 lostAndFoundBranches.add(source)
1342 for branch in lostAndFoundBranches:
1343 self.knownBranches[branch] = branch
1345 def getBranchMappingFromGitBranches(self):
1346 branches = p4BranchesInGit(self.importIntoRemotes)
1347 for branch in branches.keys():
1348 if branch == "master":
1349 branch = "main"
1350 else:
1351 branch = branch[len(self.projectName):]
1352 self.knownBranches[branch] = branch
1354 def listExistingP4GitBranches(self):
1355 # branches holds mapping from name to commit
1356 branches = p4BranchesInGit(self.importIntoRemotes)
1357 self.p4BranchesInGit = branches.keys()
1358 for branch in branches.keys():
1359 self.initialParents[self.refPrefix + branch] = branches[branch]
1361 def updateOptionDict(self, d):
1362 option_keys = {}
1363 if self.keepRepoPath:
1364 option_keys['keepRepoPath'] = 1
1366 d["options"] = ' '.join(sorted(option_keys.keys()))
1368 def readOptions(self, d):
1369 self.keepRepoPath = (d.has_key('options')
1370 and ('keepRepoPath' in d['options']))
1372 def gitRefForBranch(self, branch):
1373 if branch == "main":
1374 return self.refPrefix + "master"
1376 if len(branch) <= 0:
1377 return branch
1379 return self.refPrefix + self.projectName + branch
1381 def gitCommitByP4Change(self, ref, change):
1382 if self.verbose:
1383 print "looking in ref " + ref + " for change %s using bisect..." % change
1385 earliestCommit = ""
1386 latestCommit = parseRevision(ref)
1388 while True:
1389 if self.verbose:
1390 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1391 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1392 if len(next) == 0:
1393 if self.verbose:
1394 print "argh"
1395 return ""
1396 log = extractLogMessageFromGitCommit(next)
1397 settings = extractSettingsGitLog(log)
1398 currentChange = int(settings['change'])
1399 if self.verbose:
1400 print "current change %s" % currentChange
1402 if currentChange == change:
1403 if self.verbose:
1404 print "found %s" % next
1405 return next
1407 if currentChange < change:
1408 earliestCommit = "^%s" % next
1409 else:
1410 latestCommit = "%s" % next
1412 return ""
1414 def importNewBranch(self, branch, maxChange):
1415 # make fast-import flush all changes to disk and update the refs using the checkpoint
1416 # command so that we can try to find the branch parent in the git history
1417 self.gitStream.write("checkpoint\n\n");
1418 self.gitStream.flush();
1419 branchPrefix = self.depotPaths[0] + branch + "/"
1420 range = "@1,%s" % maxChange
1421 #print "prefix" + branchPrefix
1422 changes = p4ChangesForPaths([branchPrefix], range)
1423 if len(changes) <= 0:
1424 return False
1425 firstChange = changes[0]
1426 #print "first change in branch: %s" % firstChange
1427 sourceBranch = self.knownBranches[branch]
1428 sourceDepotPath = self.depotPaths[0] + sourceBranch
1429 sourceRef = self.gitRefForBranch(sourceBranch)
1430 #print "source " + sourceBranch
1432 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1433 #print "branch parent: %s" % branchParentChange
1434 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1435 if len(gitParent) > 0:
1436 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1437 #print "parent git commit: %s" % gitParent
1439 self.importChanges(changes)
1440 return True
1442 def importChanges(self, changes):
1443 cnt = 1
1444 for change in changes:
1445 description = p4Cmd("describe %s" % change)
1446 self.updateOptionDict(description)
1448 if not self.silent:
1449 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1450 sys.stdout.flush()
1451 cnt = cnt + 1
1453 try:
1454 if self.detectBranches:
1455 branches = self.splitFilesIntoBranches(description)
1456 for branch in branches.keys():
1457 ## HACK --hwn
1458 branchPrefix = self.depotPaths[0] + branch + "/"
1460 parent = ""
1462 filesForCommit = branches[branch]
1464 if self.verbose:
1465 print "branch is %s" % branch
1467 self.updatedBranches.add(branch)
1469 if branch not in self.createdBranches:
1470 self.createdBranches.add(branch)
1471 parent = self.knownBranches[branch]
1472 if parent == branch:
1473 parent = ""
1474 else:
1475 fullBranch = self.projectName + branch
1476 if fullBranch not in self.p4BranchesInGit:
1477 if not self.silent:
1478 print("\n Importing new branch %s" % fullBranch);
1479 if self.importNewBranch(branch, change - 1):
1480 parent = ""
1481 self.p4BranchesInGit.append(fullBranch)
1482 if not self.silent:
1483 print("\n Resuming with change %s" % change);
1485 if self.verbose:
1486 print "parent determined through known branches: %s" % parent
1488 branch = self.gitRefForBranch(branch)
1489 parent = self.gitRefForBranch(parent)
1491 if self.verbose:
1492 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1494 if len(parent) == 0 and branch in self.initialParents:
1495 parent = self.initialParents[branch]
1496 del self.initialParents[branch]
1498 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1499 else:
1500 files = self.extractFilesFromCommit(description)
1501 self.commit(description, files, self.branch, self.depotPaths,
1502 self.initialParent)
1503 self.initialParent = ""
1504 except IOError:
1505 print self.gitError.read()
1506 sys.exit(1)
1508 def importHeadRevision(self, revision):
1509 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1511 details = { "user" : "git perforce import user", "time" : int(time.time()) }
1512 details["desc"] = ("Initial import of %s from the state at revision %s\n"
1513 % (' '.join(self.depotPaths), revision))
1514 details["change"] = revision
1515 newestRevision = 0
1517 fileCnt = 0
1518 for info in p4CmdList("files "
1519 + ' '.join(["%s...%s"
1520 % (p, revision)
1521 for p in self.depotPaths])):
1523 if 'code' in info and info['code'] == 'error':
1524 sys.stderr.write("p4 returned an error: %s\n"
1525 % info['data'])
1526 if info['data'].find("must refer to client") >= 0:
1527 sys.stderr.write("This particular p4 error is misleading.\n")
1528 sys.stderr.write("Perhaps the depot path was misspelled.\n");
1529 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
1530 sys.exit(1)
1531 if 'p4ExitCode' in info:
1532 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1533 sys.exit(1)
1536 change = int(info["change"])
1537 if change > newestRevision:
1538 newestRevision = change
1540 if info["action"] in self.delete_actions:
1541 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1542 #fileCnt = fileCnt + 1
1543 continue
1545 for prop in ["depotFile", "rev", "action", "type" ]:
1546 details["%s%s" % (prop, fileCnt)] = info[prop]
1548 fileCnt = fileCnt + 1
1550 details["change"] = newestRevision
1551 self.updateOptionDict(details)
1552 try:
1553 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1554 except IOError:
1555 print "IO error with git fast-import. Is your git version recent enough?"
1556 print self.gitError.read()
1559 def getClientSpec(self):
1560 specList = p4CmdList( "client -o" )
1561 temp = {}
1562 for entry in specList:
1563 for k,v in entry.iteritems():
1564 if k.startswith("View"):
1566 # p4 has these %%1 to %%9 arguments in specs to
1567 # reorder paths; which we can't handle (yet :)
1568 if re.match('%%\d', v) != None:
1569 print "Sorry, can't handle %%n arguments in client specs"
1570 sys.exit(1)
1572 if v.startswith('"'):
1573 start = 1
1574 else:
1575 start = 0
1576 index = v.find("...")
1578 # save the "client view"; i.e the RHS of the view
1579 # line that tells the client where to put the
1580 # files for this view.
1581 cv = v[index+3:].strip() # +3 to remove previous '...'
1583 # if the client view doesn't end with a
1584 # ... wildcard, then we're going to mess up the
1585 # output directory, so fail gracefully.
1586 if not cv.endswith('...'):
1587 print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1588 sys.exit(1)
1589 cv=cv[:-3]
1591 # now save the view; +index means included, -index
1592 # means it should be filtered out.
1593 v = v[start:index]
1594 if v.startswith("-"):
1595 v = v[1:]
1596 include = -len(v)
1597 else:
1598 include = len(v)
1600 temp[v] = (include, cv)
1602 self.clientSpecDirs = temp.items()
1603 self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1605 def run(self, args):
1606 self.depotPaths = []
1607 self.changeRange = ""
1608 self.initialParent = ""
1609 self.previousDepotPaths = []
1611 # map from branch depot path to parent branch
1612 self.knownBranches = {}
1613 self.initialParents = {}
1614 self.hasOrigin = originP4BranchesExist()
1615 if not self.syncWithOrigin:
1616 self.hasOrigin = False
1618 if self.importIntoRemotes:
1619 self.refPrefix = "refs/remotes/p4/"
1620 else:
1621 self.refPrefix = "refs/heads/p4/"
1623 if self.syncWithOrigin and self.hasOrigin:
1624 if not self.silent:
1625 print "Syncing with origin first by calling git fetch origin"
1626 system("git fetch origin")
1628 if len(self.branch) == 0:
1629 self.branch = self.refPrefix + "master"
1630 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1631 system("git update-ref %s refs/heads/p4" % self.branch)
1632 system("git branch -D p4");
1633 # create it /after/ importing, when master exists
1634 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1635 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1637 if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1638 self.getClientSpec()
1640 # TODO: should always look at previous commits,
1641 # merge with previous imports, if possible.
1642 if args == []:
1643 if self.hasOrigin:
1644 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1645 self.listExistingP4GitBranches()
1647 if len(self.p4BranchesInGit) > 1:
1648 if not self.silent:
1649 print "Importing from/into multiple branches"
1650 self.detectBranches = True
1652 if self.verbose:
1653 print "branches: %s" % self.p4BranchesInGit
1655 p4Change = 0
1656 for branch in self.p4BranchesInGit:
1657 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
1659 settings = extractSettingsGitLog(logMsg)
1661 self.readOptions(settings)
1662 if (settings.has_key('depot-paths')
1663 and settings.has_key ('change')):
1664 change = int(settings['change']) + 1
1665 p4Change = max(p4Change, change)
1667 depotPaths = sorted(settings['depot-paths'])
1668 if self.previousDepotPaths == []:
1669 self.previousDepotPaths = depotPaths
1670 else:
1671 paths = []
1672 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1673 for i in range(0, min(len(cur), len(prev))):
1674 if cur[i] <> prev[i]:
1675 i = i - 1
1676 break
1678 paths.append (cur[:i + 1])
1680 self.previousDepotPaths = paths
1682 if p4Change > 0:
1683 self.depotPaths = sorted(self.previousDepotPaths)
1684 self.changeRange = "@%s,#head" % p4Change
1685 if not self.detectBranches:
1686 self.initialParent = parseRevision(self.branch)
1687 if not self.silent and not self.detectBranches:
1688 print "Performing incremental import into %s git branch" % self.branch
1690 if not self.branch.startswith("refs/"):
1691 self.branch = "refs/heads/" + self.branch
1693 if len(args) == 0 and self.depotPaths:
1694 if not self.silent:
1695 print "Depot paths: %s" % ' '.join(self.depotPaths)
1696 else:
1697 if self.depotPaths and self.depotPaths != args:
1698 print ("previous import used depot path %s and now %s was specified. "
1699 "This doesn't work!" % (' '.join (self.depotPaths),
1700 ' '.join (args)))
1701 sys.exit(1)
1703 self.depotPaths = sorted(args)
1705 revision = ""
1706 self.users = {}
1708 newPaths = []
1709 for p in self.depotPaths:
1710 if p.find("@") != -1:
1711 atIdx = p.index("@")
1712 self.changeRange = p[atIdx:]
1713 if self.changeRange == "@all":
1714 self.changeRange = ""
1715 elif ',' not in self.changeRange:
1716 revision = self.changeRange
1717 self.changeRange = ""
1718 p = p[:atIdx]
1719 elif p.find("#") != -1:
1720 hashIdx = p.index("#")
1721 revision = p[hashIdx:]
1722 p = p[:hashIdx]
1723 elif self.previousDepotPaths == []:
1724 revision = "#head"
1726 p = re.sub ("\.\.\.$", "", p)
1727 if not p.endswith("/"):
1728 p += "/"
1730 newPaths.append(p)
1732 self.depotPaths = newPaths
1735 self.loadUserMapFromCache()
1736 self.labels = {}
1737 if self.detectLabels:
1738 self.getLabels();
1740 if self.detectBranches:
1741 ## FIXME - what's a P4 projectName ?
1742 self.projectName = self.guessProjectName()
1744 if self.hasOrigin:
1745 self.getBranchMappingFromGitBranches()
1746 else:
1747 self.getBranchMapping()
1748 if self.verbose:
1749 print "p4-git branches: %s" % self.p4BranchesInGit
1750 print "initial parents: %s" % self.initialParents
1751 for b in self.p4BranchesInGit:
1752 if b != "master":
1754 ## FIXME
1755 b = b[len(self.projectName):]
1756 self.createdBranches.add(b)
1758 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1760 importProcess = subprocess.Popen(["git", "fast-import"],
1761 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1762 stderr=subprocess.PIPE);
1763 self.gitOutput = importProcess.stdout
1764 self.gitStream = importProcess.stdin
1765 self.gitError = importProcess.stderr
1767 if revision:
1768 self.importHeadRevision(revision)
1769 else:
1770 changes = []
1772 if len(self.changesFile) > 0:
1773 output = open(self.changesFile).readlines()
1774 changeSet = set()
1775 for line in output:
1776 changeSet.add(int(line))
1778 for change in changeSet:
1779 changes.append(change)
1781 changes.sort()
1782 else:
1783 # catch "git-p4 sync" with no new branches, in a repo that
1784 # does not have any existing git-p4 branches
1785 if len(args) == 0 and not self.p4BranchesInGit:
1786 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
1787 if self.verbose:
1788 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1789 self.changeRange)
1790 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1792 if len(self.maxChanges) > 0:
1793 changes = changes[:min(int(self.maxChanges), len(changes))]
1795 if len(changes) == 0:
1796 if not self.silent:
1797 print "No changes to import!"
1798 return True
1800 if not self.silent and not self.detectBranches:
1801 print "Import destination: %s" % self.branch
1803 self.updatedBranches = set()
1805 self.importChanges(changes)
1807 if not self.silent:
1808 print ""
1809 if len(self.updatedBranches) > 0:
1810 sys.stdout.write("Updated branches: ")
1811 for b in self.updatedBranches:
1812 sys.stdout.write("%s " % b)
1813 sys.stdout.write("\n")
1815 self.gitStream.close()
1816 if importProcess.wait() != 0:
1817 die("fast-import failed: %s" % self.gitError.read())
1818 self.gitOutput.close()
1819 self.gitError.close()
1821 return True
1823 class P4Rebase(Command):
1824 def __init__(self):
1825 Command.__init__(self)
1826 self.options = [ ]
1827 self.description = ("Fetches the latest revision from perforce and "
1828 + "rebases the current work (branch) against it")
1829 self.verbose = False
1831 def run(self, args):
1832 sync = P4Sync()
1833 sync.run([])
1835 return self.rebase()
1837 def rebase(self):
1838 if os.system("git update-index --refresh") != 0:
1839 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1840 if len(read_pipe("git diff-index HEAD --")) > 0:
1841 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1843 [upstream, settings] = findUpstreamBranchPoint()
1844 if len(upstream) == 0:
1845 die("Cannot find upstream branchpoint for rebase")
1847 # the branchpoint may be p4/foo~3, so strip off the parent
1848 upstream = re.sub("~[0-9]+$", "", upstream)
1850 print "Rebasing the current branch onto %s" % upstream
1851 oldHead = read_pipe("git rev-parse HEAD").strip()
1852 system("git rebase %s" % upstream)
1853 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1854 return True
1856 class P4Clone(P4Sync):
1857 def __init__(self):
1858 P4Sync.__init__(self)
1859 self.description = "Creates a new git repository and imports from Perforce into it"
1860 self.usage = "usage: %prog [options] //depot/path[@revRange]"
1861 self.options += [
1862 optparse.make_option("--destination", dest="cloneDestination",
1863 action='store', default=None,
1864 help="where to leave result of the clone"),
1865 optparse.make_option("-/", dest="cloneExclude",
1866 action="append", type="string",
1867 help="exclude depot path"),
1868 optparse.make_option("--bare", dest="cloneBare",
1869 action="store_true", default=False),
1871 self.cloneDestination = None
1872 self.needsGit = False
1873 self.cloneBare = False
1875 # This is required for the "append" cloneExclude action
1876 def ensure_value(self, attr, value):
1877 if not hasattr(self, attr) or getattr(self, attr) is None:
1878 setattr(self, attr, value)
1879 return getattr(self, attr)
1881 def defaultDestination(self, args):
1882 ## TODO: use common prefix of args?
1883 depotPath = args[0]
1884 depotDir = re.sub("(@[^@]*)$", "", depotPath)
1885 depotDir = re.sub("(#[^#]*)$", "", depotDir)
1886 depotDir = re.sub(r"\.\.\.$", "", depotDir)
1887 depotDir = re.sub(r"/$", "", depotDir)
1888 return os.path.split(depotDir)[1]
1890 def run(self, args):
1891 if len(args) < 1:
1892 return False
1894 if self.keepRepoPath and not self.cloneDestination:
1895 sys.stderr.write("Must specify destination for --keep-path\n")
1896 sys.exit(1)
1898 depotPaths = args
1900 if not self.cloneDestination and len(depotPaths) > 1:
1901 self.cloneDestination = depotPaths[-1]
1902 depotPaths = depotPaths[:-1]
1904 self.cloneExclude = ["/"+p for p in self.cloneExclude]
1905 for p in depotPaths:
1906 if not p.startswith("//"):
1907 return False
1909 if not self.cloneDestination:
1910 self.cloneDestination = self.defaultDestination(args)
1912 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1914 if not os.path.exists(self.cloneDestination):
1915 os.makedirs(self.cloneDestination)
1916 chdir(self.cloneDestination)
1918 init_cmd = [ "git", "init" ]
1919 if self.cloneBare:
1920 init_cmd.append("--bare")
1921 subprocess.check_call(init_cmd)
1923 if not P4Sync.run(self, depotPaths):
1924 return False
1925 if self.branch != "master":
1926 if self.importIntoRemotes:
1927 masterbranch = "refs/remotes/p4/master"
1928 else:
1929 masterbranch = "refs/heads/p4/master"
1930 if gitBranchExists(masterbranch):
1931 system("git branch master %s" % masterbranch)
1932 if not self.cloneBare:
1933 system("git checkout -f")
1934 else:
1935 print "Could not detect main branch. No checkout/master branch created."
1937 return True
1939 class P4Branches(Command):
1940 def __init__(self):
1941 Command.__init__(self)
1942 self.options = [ ]
1943 self.description = ("Shows the git branches that hold imports and their "
1944 + "corresponding perforce depot paths")
1945 self.verbose = False
1947 def run(self, args):
1948 if originP4BranchesExist():
1949 createOrUpdateBranchesFromOrigin()
1951 cmdline = "git rev-parse --symbolic "
1952 cmdline += " --remotes"
1954 for line in read_pipe_lines(cmdline):
1955 line = line.strip()
1957 if not line.startswith('p4/') or line == "p4/HEAD":
1958 continue
1959 branch = line
1961 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1962 settings = extractSettingsGitLog(log)
1964 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1965 return True
1967 class HelpFormatter(optparse.IndentedHelpFormatter):
1968 def __init__(self):
1969 optparse.IndentedHelpFormatter.__init__(self)
1971 def format_description(self, description):
1972 if description:
1973 return description + "\n"
1974 else:
1975 return ""
1977 def printUsage(commands):
1978 print "usage: %s <command> [options]" % sys.argv[0]
1979 print ""
1980 print "valid commands: %s" % ", ".join(commands)
1981 print ""
1982 print "Try %s <command> --help for command specific help." % sys.argv[0]
1983 print ""
1985 commands = {
1986 "debug" : P4Debug,
1987 "submit" : P4Submit,
1988 "commit" : P4Submit,
1989 "sync" : P4Sync,
1990 "rebase" : P4Rebase,
1991 "clone" : P4Clone,
1992 "rollback" : P4RollBack,
1993 "branches" : P4Branches
1997 def main():
1998 if len(sys.argv[1:]) == 0:
1999 printUsage(commands.keys())
2000 sys.exit(2)
2002 cmd = ""
2003 cmdName = sys.argv[1]
2004 try:
2005 klass = commands[cmdName]
2006 cmd = klass()
2007 except KeyError:
2008 print "unknown command %s" % cmdName
2009 print ""
2010 printUsage(commands.keys())
2011 sys.exit(2)
2013 options = cmd.options
2014 cmd.gitdir = os.environ.get("GIT_DIR", None)
2016 args = sys.argv[2:]
2018 if len(options) > 0:
2019 options.append(optparse.make_option("--git-dir", dest="gitdir"))
2021 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2022 options,
2023 description = cmd.description,
2024 formatter = HelpFormatter())
2026 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2027 global verbose
2028 verbose = cmd.verbose
2029 if cmd.needsGit:
2030 if cmd.gitdir == None:
2031 cmd.gitdir = os.path.abspath(".git")
2032 if not isValidGitDir(cmd.gitdir):
2033 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2034 if os.path.exists(cmd.gitdir):
2035 cdup = read_pipe("git rev-parse --show-cdup").strip()
2036 if len(cdup) > 0:
2037 chdir(cdup);
2039 if not isValidGitDir(cmd.gitdir):
2040 if isValidGitDir(cmd.gitdir + "/.git"):
2041 cmd.gitdir += "/.git"
2042 else:
2043 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2045 os.environ["GIT_DIR"] = cmd.gitdir
2047 if not cmd.run(args):
2048 parser.print_help()
2051 if __name__ == '__main__':
2052 main()