Merge branch 'jc/maint-fetch-tighten-refname-check'
[git/jrn.git] / git-p4.py
blob882b1bbab53c6e184c7eadb5250c128b59c93482
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse, sys, os, marshal, subprocess, shelve
12 import tempfile, getopt, os.path, time, platform
13 import re, shutil
15 verbose = False
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
26 """
27 real_cmd = ["p4"]
29 user = gitConfig("git-p4.user")
30 if len(user) > 0:
31 real_cmd += ["-u",user]
33 password = gitConfig("git-p4.password")
34 if len(password) > 0:
35 real_cmd += ["-P", password]
37 port = gitConfig("git-p4.port")
38 if len(port) > 0:
39 real_cmd += ["-p", port]
41 host = gitConfig("git-p4.host")
42 if len(host) > 0:
43 real_cmd += ["-H", host]
45 client = gitConfig("git-p4.client")
46 if len(client) > 0:
47 real_cmd += ["-c", client]
50 if isinstance(cmd,basestring):
51 real_cmd = ' '.join(real_cmd) + ' ' + cmd
52 else:
53 real_cmd += cmd
54 return real_cmd
56 def chdir(dir):
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
60 os.chdir(dir)
61 os.environ['PWD'] = os.getcwd()
63 def die(msg):
64 if verbose:
65 raise Exception(msg)
66 else:
67 sys.stderr.write(msg + "\n")
68 sys.exit(1)
70 def write_pipe(c, stdin):
71 if verbose:
72 sys.stderr.write('Writing pipe: %s\n' % str(c))
74 expand = isinstance(c,basestring)
75 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
76 pipe = p.stdin
77 val = pipe.write(stdin)
78 pipe.close()
79 if p.wait():
80 die('Command failed: %s' % str(c))
82 return val
84 def p4_write_pipe(c, stdin):
85 real_cmd = p4_build_cmd(c)
86 return write_pipe(real_cmd, stdin)
88 def read_pipe(c, ignore_error=False):
89 if verbose:
90 sys.stderr.write('Reading pipe: %s\n' % str(c))
92 expand = isinstance(c,basestring)
93 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
94 pipe = p.stdout
95 val = pipe.read()
96 if p.wait() and not ignore_error:
97 die('Command failed: %s' % str(c))
99 return val
101 def p4_read_pipe(c, ignore_error=False):
102 real_cmd = p4_build_cmd(c)
103 return read_pipe(real_cmd, ignore_error)
105 def read_pipe_lines(c):
106 if verbose:
107 sys.stderr.write('Reading pipe: %s\n' % str(c))
109 expand = isinstance(c, basestring)
110 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
111 pipe = p.stdout
112 val = pipe.readlines()
113 if pipe.close() or p.wait():
114 die('Command failed: %s' % str(c))
116 return val
118 def p4_read_pipe_lines(c):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd = p4_build_cmd(c)
121 return read_pipe_lines(real_cmd)
123 def p4_has_command(cmd):
124 """Ask p4 for help on this command. If it returns an error, the
125 command does not exist in this version of p4."""
126 real_cmd = p4_build_cmd(["help", cmd])
127 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
128 stderr=subprocess.PIPE)
129 p.communicate()
130 return p.returncode == 0
132 def system(cmd):
133 expand = isinstance(cmd,basestring)
134 if verbose:
135 sys.stderr.write("executing %s\n" % str(cmd))
136 subprocess.check_call(cmd, shell=expand)
138 def p4_system(cmd):
139 """Specifically invoke p4 as the system command. """
140 real_cmd = p4_build_cmd(cmd)
141 expand = isinstance(real_cmd, basestring)
142 subprocess.check_call(real_cmd, shell=expand)
144 def p4_integrate(src, dest):
145 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
147 def p4_sync(f, *options):
148 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
150 def p4_add(f):
151 # forcibly add file names with wildcards
152 if wildcard_present(f):
153 p4_system(["add", "-f", f])
154 else:
155 p4_system(["add", f])
157 def p4_delete(f):
158 p4_system(["delete", wildcard_encode(f)])
160 def p4_edit(f):
161 p4_system(["edit", wildcard_encode(f)])
163 def p4_revert(f):
164 p4_system(["revert", wildcard_encode(f)])
166 def p4_reopen(type, f):
167 p4_system(["reopen", "-t", type, wildcard_encode(f)])
169 def p4_move(src, dest):
170 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
173 # Canonicalize the p4 type and return a tuple of the
174 # base type, plus any modifiers. See "p4 help filetypes"
175 # for a list and explanation.
177 def split_p4_type(p4type):
179 p4_filetypes_historical = {
180 "ctempobj": "binary+Sw",
181 "ctext": "text+C",
182 "cxtext": "text+Cx",
183 "ktext": "text+k",
184 "kxtext": "text+kx",
185 "ltext": "text+F",
186 "tempobj": "binary+FSw",
187 "ubinary": "binary+F",
188 "uresource": "resource+F",
189 "uxbinary": "binary+Fx",
190 "xbinary": "binary+x",
191 "xltext": "text+Fx",
192 "xtempobj": "binary+Swx",
193 "xtext": "text+x",
194 "xunicode": "unicode+x",
195 "xutf16": "utf16+x",
197 if p4type in p4_filetypes_historical:
198 p4type = p4_filetypes_historical[p4type]
199 mods = ""
200 s = p4type.split("+")
201 base = s[0]
202 mods = ""
203 if len(s) > 1:
204 mods = s[1]
205 return (base, mods)
208 # return the raw p4 type of a file (text, text+ko, etc)
210 def p4_type(file):
211 results = p4CmdList(["fstat", "-T", "headType", file])
212 return results[0]['headType']
215 # Given a type base and modifier, return a regexp matching
216 # the keywords that can be expanded in the file
218 def p4_keywords_regexp_for_type(base, type_mods):
219 if base in ("text", "unicode", "binary"):
220 kwords = None
221 if "ko" in type_mods:
222 kwords = 'Id|Header'
223 elif "k" in type_mods:
224 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
225 else:
226 return None
227 pattern = r"""
228 \$ # Starts with a dollar, followed by...
229 (%s) # one of the keywords, followed by...
230 (:[^$]+)? # possibly an old expansion, followed by...
231 \$ # another dollar
232 """ % kwords
233 return pattern
234 else:
235 return None
238 # Given a file, return a regexp matching the possible
239 # RCS keywords that will be expanded, or None for files
240 # with kw expansion turned off.
242 def p4_keywords_regexp_for_file(file):
243 if not os.path.exists(file):
244 return None
245 else:
246 (type_base, type_mods) = split_p4_type(p4_type(file))
247 return p4_keywords_regexp_for_type(type_base, type_mods)
249 def setP4ExecBit(file, mode):
250 # Reopens an already open file and changes the execute bit to match
251 # the execute bit setting in the passed in mode.
253 p4Type = "+x"
255 if not isModeExec(mode):
256 p4Type = getP4OpenedType(file)
257 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
258 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
259 if p4Type[-1] == "+":
260 p4Type = p4Type[0:-1]
262 p4_reopen(p4Type, file)
264 def getP4OpenedType(file):
265 # Returns the perforce file type for the given file.
267 result = p4_read_pipe(["opened", wildcard_encode(file)])
268 match = re.match(".*\((.+)\)\r?$", result)
269 if match:
270 return match.group(1)
271 else:
272 die("Could not determine file type for %s (result: '%s')" % (file, result))
274 # Return the set of all p4 labels
275 def getP4Labels(depotPaths):
276 labels = set()
277 if isinstance(depotPaths,basestring):
278 depotPaths = [depotPaths]
280 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
281 label = l['label']
282 labels.add(label)
284 return labels
286 # Return the set of all git tags
287 def getGitTags():
288 gitTags = set()
289 for line in read_pipe_lines(["git", "tag"]):
290 tag = line.strip()
291 gitTags.add(tag)
292 return gitTags
294 def diffTreePattern():
295 # This is a simple generator for the diff tree regex pattern. This could be
296 # a class variable if this and parseDiffTreeEntry were a part of a class.
297 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
298 while True:
299 yield pattern
301 def parseDiffTreeEntry(entry):
302 """Parses a single diff tree entry into its component elements.
304 See git-diff-tree(1) manpage for details about the format of the diff
305 output. This method returns a dictionary with the following elements:
307 src_mode - The mode of the source file
308 dst_mode - The mode of the destination file
309 src_sha1 - The sha1 for the source file
310 dst_sha1 - The sha1 fr the destination file
311 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
312 status_score - The score for the status (applicable for 'C' and 'R'
313 statuses). This is None if there is no score.
314 src - The path for the source file.
315 dst - The path for the destination file. This is only present for
316 copy or renames. If it is not present, this is None.
318 If the pattern is not matched, None is returned."""
320 match = diffTreePattern().next().match(entry)
321 if match:
322 return {
323 'src_mode': match.group(1),
324 'dst_mode': match.group(2),
325 'src_sha1': match.group(3),
326 'dst_sha1': match.group(4),
327 'status': match.group(5),
328 'status_score': match.group(6),
329 'src': match.group(7),
330 'dst': match.group(10)
332 return None
334 def isModeExec(mode):
335 # Returns True if the given git mode represents an executable file,
336 # otherwise False.
337 return mode[-3:] == "755"
339 def isModeExecChanged(src_mode, dst_mode):
340 return isModeExec(src_mode) != isModeExec(dst_mode)
342 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
344 if isinstance(cmd,basestring):
345 cmd = "-G " + cmd
346 expand = True
347 else:
348 cmd = ["-G"] + cmd
349 expand = False
351 cmd = p4_build_cmd(cmd)
352 if verbose:
353 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
355 # Use a temporary file to avoid deadlocks without
356 # subprocess.communicate(), which would put another copy
357 # of stdout into memory.
358 stdin_file = None
359 if stdin is not None:
360 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
361 if isinstance(stdin,basestring):
362 stdin_file.write(stdin)
363 else:
364 for i in stdin:
365 stdin_file.write(i + '\n')
366 stdin_file.flush()
367 stdin_file.seek(0)
369 p4 = subprocess.Popen(cmd,
370 shell=expand,
371 stdin=stdin_file,
372 stdout=subprocess.PIPE)
374 result = []
375 try:
376 while True:
377 entry = marshal.load(p4.stdout)
378 if cb is not None:
379 cb(entry)
380 else:
381 result.append(entry)
382 except EOFError:
383 pass
384 exitCode = p4.wait()
385 if exitCode != 0:
386 entry = {}
387 entry["p4ExitCode"] = exitCode
388 result.append(entry)
390 return result
392 def p4Cmd(cmd):
393 list = p4CmdList(cmd)
394 result = {}
395 for entry in list:
396 result.update(entry)
397 return result;
399 def p4Where(depotPath):
400 if not depotPath.endswith("/"):
401 depotPath += "/"
402 depotPath = depotPath + "..."
403 outputList = p4CmdList(["where", depotPath])
404 output = None
405 for entry in outputList:
406 if "depotFile" in entry:
407 if entry["depotFile"] == depotPath:
408 output = entry
409 break
410 elif "data" in entry:
411 data = entry.get("data")
412 space = data.find(" ")
413 if data[:space] == depotPath:
414 output = entry
415 break
416 if output == None:
417 return ""
418 if output["code"] == "error":
419 return ""
420 clientPath = ""
421 if "path" in output:
422 clientPath = output.get("path")
423 elif "data" in output:
424 data = output.get("data")
425 lastSpace = data.rfind(" ")
426 clientPath = data[lastSpace + 1:]
428 if clientPath.endswith("..."):
429 clientPath = clientPath[:-3]
430 return clientPath
432 def currentGitBranch():
433 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
435 def isValidGitDir(path):
436 if (os.path.exists(path + "/HEAD")
437 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
438 return True;
439 return False
441 def parseRevision(ref):
442 return read_pipe("git rev-parse %s" % ref).strip()
444 def branchExists(ref):
445 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
446 ignore_error=True)
447 return len(rev) > 0
449 def extractLogMessageFromGitCommit(commit):
450 logMessage = ""
452 ## fixme: title is first line of commit, not 1st paragraph.
453 foundTitle = False
454 for log in read_pipe_lines("git cat-file commit %s" % commit):
455 if not foundTitle:
456 if len(log) == 1:
457 foundTitle = True
458 continue
460 logMessage += log
461 return logMessage
463 def extractSettingsGitLog(log):
464 values = {}
465 for line in log.split("\n"):
466 line = line.strip()
467 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
468 if not m:
469 continue
471 assignments = m.group(1).split (':')
472 for a in assignments:
473 vals = a.split ('=')
474 key = vals[0].strip()
475 val = ('='.join (vals[1:])).strip()
476 if val.endswith ('\"') and val.startswith('"'):
477 val = val[1:-1]
479 values[key] = val
481 paths = values.get("depot-paths")
482 if not paths:
483 paths = values.get("depot-path")
484 if paths:
485 values['depot-paths'] = paths.split(',')
486 return values
488 def gitBranchExists(branch):
489 proc = subprocess.Popen(["git", "rev-parse", branch],
490 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
491 return proc.wait() == 0;
493 _gitConfig = {}
494 def gitConfig(key, args = None): # set args to "--bool", for instance
495 if not _gitConfig.has_key(key):
496 argsFilter = ""
497 if args != None:
498 argsFilter = "%s " % args
499 cmd = "git config %s%s" % (argsFilter, key)
500 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
501 return _gitConfig[key]
503 def gitConfigList(key):
504 if not _gitConfig.has_key(key):
505 _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
506 return _gitConfig[key]
508 def p4BranchesInGit(branchesAreInRemotes = True):
509 branches = {}
511 cmdline = "git rev-parse --symbolic "
512 if branchesAreInRemotes:
513 cmdline += " --remotes"
514 else:
515 cmdline += " --branches"
517 for line in read_pipe_lines(cmdline):
518 line = line.strip()
520 ## only import to p4/
521 if not line.startswith('p4/') or line == "p4/HEAD":
522 continue
523 branch = line
525 # strip off p4
526 branch = re.sub ("^p4/", "", line)
528 branches[branch] = parseRevision(line)
529 return branches
531 def findUpstreamBranchPoint(head = "HEAD"):
532 branches = p4BranchesInGit()
533 # map from depot-path to branch name
534 branchByDepotPath = {}
535 for branch in branches.keys():
536 tip = branches[branch]
537 log = extractLogMessageFromGitCommit(tip)
538 settings = extractSettingsGitLog(log)
539 if settings.has_key("depot-paths"):
540 paths = ",".join(settings["depot-paths"])
541 branchByDepotPath[paths] = "remotes/p4/" + branch
543 settings = None
544 parent = 0
545 while parent < 65535:
546 commit = head + "~%s" % parent
547 log = extractLogMessageFromGitCommit(commit)
548 settings = extractSettingsGitLog(log)
549 if settings.has_key("depot-paths"):
550 paths = ",".join(settings["depot-paths"])
551 if branchByDepotPath.has_key(paths):
552 return [branchByDepotPath[paths], settings]
554 parent = parent + 1
556 return ["", settings]
558 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
559 if not silent:
560 print ("Creating/updating branch(es) in %s based on origin branch(es)"
561 % localRefPrefix)
563 originPrefix = "origin/p4/"
565 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
566 line = line.strip()
567 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
568 continue
570 headName = line[len(originPrefix):]
571 remoteHead = localRefPrefix + headName
572 originHead = line
574 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
575 if (not original.has_key('depot-paths')
576 or not original.has_key('change')):
577 continue
579 update = False
580 if not gitBranchExists(remoteHead):
581 if verbose:
582 print "creating %s" % remoteHead
583 update = True
584 else:
585 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
586 if settings.has_key('change') > 0:
587 if settings['depot-paths'] == original['depot-paths']:
588 originP4Change = int(original['change'])
589 p4Change = int(settings['change'])
590 if originP4Change > p4Change:
591 print ("%s (%s) is newer than %s (%s). "
592 "Updating p4 branch from origin."
593 % (originHead, originP4Change,
594 remoteHead, p4Change))
595 update = True
596 else:
597 print ("Ignoring: %s was imported from %s while "
598 "%s was imported from %s"
599 % (originHead, ','.join(original['depot-paths']),
600 remoteHead, ','.join(settings['depot-paths'])))
602 if update:
603 system("git update-ref %s %s" % (remoteHead, originHead))
605 def originP4BranchesExist():
606 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
608 def p4ChangesForPaths(depotPaths, changeRange):
609 assert depotPaths
610 cmd = ['changes']
611 for p in depotPaths:
612 cmd += ["%s...%s" % (p, changeRange)]
613 output = p4_read_pipe_lines(cmd)
615 changes = {}
616 for line in output:
617 changeNum = int(line.split(" ")[1])
618 changes[changeNum] = True
620 changelist = changes.keys()
621 changelist.sort()
622 return changelist
624 def p4PathStartsWith(path, prefix):
625 # This method tries to remedy a potential mixed-case issue:
627 # If UserA adds //depot/DirA/file1
628 # and UserB adds //depot/dira/file2
630 # we may or may not have a problem. If you have core.ignorecase=true,
631 # we treat DirA and dira as the same directory
632 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
633 if ignorecase:
634 return path.lower().startswith(prefix.lower())
635 return path.startswith(prefix)
637 def getClientSpec():
638 """Look at the p4 client spec, create a View() object that contains
639 all the mappings, and return it."""
641 specList = p4CmdList("client -o")
642 if len(specList) != 1:
643 die('Output from "client -o" is %d lines, expecting 1' %
644 len(specList))
646 # dictionary of all client parameters
647 entry = specList[0]
649 # just the keys that start with "View"
650 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
652 # hold this new View
653 view = View()
655 # append the lines, in order, to the view
656 for view_num in range(len(view_keys)):
657 k = "View%d" % view_num
658 if k not in view_keys:
659 die("Expected view key %s missing" % k)
660 view.append(entry[k])
662 return view
664 def getClientRoot():
665 """Grab the client directory."""
667 output = p4CmdList("client -o")
668 if len(output) != 1:
669 die('Output from "client -o" is %d lines, expecting 1' % len(output))
671 entry = output[0]
672 if "Root" not in entry:
673 die('Client has no "Root"')
675 return entry["Root"]
678 # P4 wildcards are not allowed in filenames. P4 complains
679 # if you simply add them, but you can force it with "-f", in
680 # which case it translates them into %xx encoding internally.
682 def wildcard_decode(path):
683 # Search for and fix just these four characters. Do % last so
684 # that fixing it does not inadvertently create new %-escapes.
685 # Cannot have * in a filename in windows; untested as to
686 # what p4 would do in such a case.
687 if not platform.system() == "Windows":
688 path = path.replace("%2A", "*")
689 path = path.replace("%23", "#") \
690 .replace("%40", "@") \
691 .replace("%25", "%")
692 return path
694 def wildcard_encode(path):
695 # do % first to avoid double-encoding the %s introduced here
696 path = path.replace("%", "%25") \
697 .replace("*", "%2A") \
698 .replace("#", "%23") \
699 .replace("@", "%40")
700 return path
702 def wildcard_present(path):
703 return path.translate(None, "*#@%") != path
705 class Command:
706 def __init__(self):
707 self.usage = "usage: %prog [options]"
708 self.needsGit = True
709 self.verbose = False
711 class P4UserMap:
712 def __init__(self):
713 self.userMapFromPerforceServer = False
714 self.myP4UserId = None
716 def p4UserId(self):
717 if self.myP4UserId:
718 return self.myP4UserId
720 results = p4CmdList("user -o")
721 for r in results:
722 if r.has_key('User'):
723 self.myP4UserId = r['User']
724 return r['User']
725 die("Could not find your p4 user id")
727 def p4UserIsMe(self, p4User):
728 # return True if the given p4 user is actually me
729 me = self.p4UserId()
730 if not p4User or p4User != me:
731 return False
732 else:
733 return True
735 def getUserCacheFilename(self):
736 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
737 return home + "/.gitp4-usercache.txt"
739 def getUserMapFromPerforceServer(self):
740 if self.userMapFromPerforceServer:
741 return
742 self.users = {}
743 self.emails = {}
745 for output in p4CmdList("users"):
746 if not output.has_key("User"):
747 continue
748 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
749 self.emails[output["Email"]] = output["User"]
752 s = ''
753 for (key, val) in self.users.items():
754 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
756 open(self.getUserCacheFilename(), "wb").write(s)
757 self.userMapFromPerforceServer = True
759 def loadUserMapFromCache(self):
760 self.users = {}
761 self.userMapFromPerforceServer = False
762 try:
763 cache = open(self.getUserCacheFilename(), "rb")
764 lines = cache.readlines()
765 cache.close()
766 for line in lines:
767 entry = line.strip().split("\t")
768 self.users[entry[0]] = entry[1]
769 except IOError:
770 self.getUserMapFromPerforceServer()
772 class P4Debug(Command):
773 def __init__(self):
774 Command.__init__(self)
775 self.options = []
776 self.description = "A tool to debug the output of p4 -G."
777 self.needsGit = False
779 def run(self, args):
780 j = 0
781 for output in p4CmdList(args):
782 print 'Element: %d' % j
783 j += 1
784 print output
785 return True
787 class P4RollBack(Command):
788 def __init__(self):
789 Command.__init__(self)
790 self.options = [
791 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
793 self.description = "A tool to debug the multi-branch import. Don't use :)"
794 self.rollbackLocalBranches = False
796 def run(self, args):
797 if len(args) != 1:
798 return False
799 maxChange = int(args[0])
801 if "p4ExitCode" in p4Cmd("changes -m 1"):
802 die("Problems executing p4");
804 if self.rollbackLocalBranches:
805 refPrefix = "refs/heads/"
806 lines = read_pipe_lines("git rev-parse --symbolic --branches")
807 else:
808 refPrefix = "refs/remotes/"
809 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
811 for line in lines:
812 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
813 line = line.strip()
814 ref = refPrefix + line
815 log = extractLogMessageFromGitCommit(ref)
816 settings = extractSettingsGitLog(log)
818 depotPaths = settings['depot-paths']
819 change = settings['change']
821 changed = False
823 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
824 for p in depotPaths]))) == 0:
825 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
826 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
827 continue
829 while change and int(change) > maxChange:
830 changed = True
831 if self.verbose:
832 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
833 system("git update-ref %s \"%s^\"" % (ref, ref))
834 log = extractLogMessageFromGitCommit(ref)
835 settings = extractSettingsGitLog(log)
838 depotPaths = settings['depot-paths']
839 change = settings['change']
841 if changed:
842 print "%s rewound to %s" % (ref, change)
844 return True
846 class P4Submit(Command, P4UserMap):
848 conflict_behavior_choices = ("ask", "skip", "quit")
850 def __init__(self):
851 Command.__init__(self)
852 P4UserMap.__init__(self)
853 self.options = [
854 optparse.make_option("--origin", dest="origin"),
855 optparse.make_option("-M", dest="detectRenames", action="store_true"),
856 # preserve the user, requires relevant p4 permissions
857 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
858 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
859 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
860 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
861 optparse.make_option("--conflict", dest="conflict_behavior",
862 choices=self.conflict_behavior_choices)
864 self.description = "Submit changes from git to the perforce depot."
865 self.usage += " [name of git branch to submit into perforce depot]"
866 self.origin = ""
867 self.detectRenames = False
868 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
869 self.dry_run = False
870 self.prepare_p4_only = False
871 self.conflict_behavior = None
872 self.isWindows = (platform.system() == "Windows")
873 self.exportLabels = False
874 self.p4HasMoveCommand = p4_has_command("move")
876 def check(self):
877 if len(p4CmdList("opened ...")) > 0:
878 die("You have files opened with perforce! Close them before starting the sync.")
880 def separate_jobs_from_description(self, message):
881 """Extract and return a possible Jobs field in the commit
882 message. It goes into a separate section in the p4 change
883 specification.
885 A jobs line starts with "Jobs:" and looks like a new field
886 in a form. Values are white-space separated on the same
887 line or on following lines that start with a tab.
889 This does not parse and extract the full git commit message
890 like a p4 form. It just sees the Jobs: line as a marker
891 to pass everything from then on directly into the p4 form,
892 but outside the description section.
894 Return a tuple (stripped log message, jobs string)."""
896 m = re.search(r'^Jobs:', message, re.MULTILINE)
897 if m is None:
898 return (message, None)
900 jobtext = message[m.start():]
901 stripped_message = message[:m.start()].rstrip()
902 return (stripped_message, jobtext)
904 def prepareLogMessage(self, template, message, jobs):
905 """Edits the template returned from "p4 change -o" to insert
906 the message in the Description field, and the jobs text in
907 the Jobs field."""
908 result = ""
910 inDescriptionSection = False
912 for line in template.split("\n"):
913 if line.startswith("#"):
914 result += line + "\n"
915 continue
917 if inDescriptionSection:
918 if line.startswith("Files:") or line.startswith("Jobs:"):
919 inDescriptionSection = False
920 # insert Jobs section
921 if jobs:
922 result += jobs + "\n"
923 else:
924 continue
925 else:
926 if line.startswith("Description:"):
927 inDescriptionSection = True
928 line += "\n"
929 for messageLine in message.split("\n"):
930 line += "\t" + messageLine + "\n"
932 result += line + "\n"
934 return result
936 def patchRCSKeywords(self, file, pattern):
937 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
938 (handle, outFileName) = tempfile.mkstemp(dir='.')
939 try:
940 outFile = os.fdopen(handle, "w+")
941 inFile = open(file, "r")
942 regexp = re.compile(pattern, re.VERBOSE)
943 for line in inFile.readlines():
944 line = regexp.sub(r'$\1$', line)
945 outFile.write(line)
946 inFile.close()
947 outFile.close()
948 # Forcibly overwrite the original file
949 os.unlink(file)
950 shutil.move(outFileName, file)
951 except:
952 # cleanup our temporary file
953 os.unlink(outFileName)
954 print "Failed to strip RCS keywords in %s" % file
955 raise
957 print "Patched up RCS keywords in %s" % file
959 def p4UserForCommit(self,id):
960 # Return the tuple (perforce user,git email) for a given git commit id
961 self.getUserMapFromPerforceServer()
962 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
963 gitEmail = gitEmail.strip()
964 if not self.emails.has_key(gitEmail):
965 return (None,gitEmail)
966 else:
967 return (self.emails[gitEmail],gitEmail)
969 def checkValidP4Users(self,commits):
970 # check if any git authors cannot be mapped to p4 users
971 for id in commits:
972 (user,email) = self.p4UserForCommit(id)
973 if not user:
974 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
975 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
976 print "%s" % msg
977 else:
978 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
980 def lastP4Changelist(self):
981 # Get back the last changelist number submitted in this client spec. This
982 # then gets used to patch up the username in the change. If the same
983 # client spec is being used by multiple processes then this might go
984 # wrong.
985 results = p4CmdList("client -o") # find the current client
986 client = None
987 for r in results:
988 if r.has_key('Client'):
989 client = r['Client']
990 break
991 if not client:
992 die("could not get client spec")
993 results = p4CmdList(["changes", "-c", client, "-m", "1"])
994 for r in results:
995 if r.has_key('change'):
996 return r['change']
997 die("Could not get changelist number for last submit - cannot patch up user details")
999 def modifyChangelistUser(self, changelist, newUser):
1000 # fixup the user field of a changelist after it has been submitted.
1001 changes = p4CmdList("change -o %s" % changelist)
1002 if len(changes) != 1:
1003 die("Bad output from p4 change modifying %s to user %s" %
1004 (changelist, newUser))
1006 c = changes[0]
1007 if c['User'] == newUser: return # nothing to do
1008 c['User'] = newUser
1009 input = marshal.dumps(c)
1011 result = p4CmdList("change -f -i", stdin=input)
1012 for r in result:
1013 if r.has_key('code'):
1014 if r['code'] == 'error':
1015 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1016 if r.has_key('data'):
1017 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1018 return
1019 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1021 def canChangeChangelists(self):
1022 # check to see if we have p4 admin or super-user permissions, either of
1023 # which are required to modify changelists.
1024 results = p4CmdList(["protects", self.depotPath])
1025 for r in results:
1026 if r.has_key('perm'):
1027 if r['perm'] == 'admin':
1028 return 1
1029 if r['perm'] == 'super':
1030 return 1
1031 return 0
1033 def prepareSubmitTemplate(self):
1034 """Run "p4 change -o" to grab a change specification template.
1035 This does not use "p4 -G", as it is nice to keep the submission
1036 template in original order, since a human might edit it.
1038 Remove lines in the Files section that show changes to files
1039 outside the depot path we're committing into."""
1041 template = ""
1042 inFilesSection = False
1043 for line in p4_read_pipe_lines(['change', '-o']):
1044 if line.endswith("\r\n"):
1045 line = line[:-2] + "\n"
1046 if inFilesSection:
1047 if line.startswith("\t"):
1048 # path starts and ends with a tab
1049 path = line[1:]
1050 lastTab = path.rfind("\t")
1051 if lastTab != -1:
1052 path = path[:lastTab]
1053 if not p4PathStartsWith(path, self.depotPath):
1054 continue
1055 else:
1056 inFilesSection = False
1057 else:
1058 if line.startswith("Files:"):
1059 inFilesSection = True
1061 template += line
1063 return template
1065 def edit_template(self, template_file):
1066 """Invoke the editor to let the user change the submission
1067 message. Return true if okay to continue with the submit."""
1069 # if configured to skip the editing part, just submit
1070 if gitConfig("git-p4.skipSubmitEdit") == "true":
1071 return True
1073 # look at the modification time, to check later if the user saved
1074 # the file
1075 mtime = os.stat(template_file).st_mtime
1077 # invoke the editor
1078 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1079 editor = os.environ.get("P4EDITOR")
1080 else:
1081 editor = read_pipe("git var GIT_EDITOR").strip()
1082 system(editor + " " + template_file)
1084 # If the file was not saved, prompt to see if this patch should
1085 # be skipped. But skip this verification step if configured so.
1086 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1087 return True
1089 # modification time updated means user saved the file
1090 if os.stat(template_file).st_mtime > mtime:
1091 return True
1093 while True:
1094 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1095 if response == 'y':
1096 return True
1097 if response == 'n':
1098 return False
1100 def applyCommit(self, id):
1101 """Apply one commit, return True if it succeeded."""
1103 print "Applying", read_pipe(["git", "show", "-s",
1104 "--format=format:%h %s", id])
1106 (p4User, gitEmail) = self.p4UserForCommit(id)
1108 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1109 filesToAdd = set()
1110 filesToDelete = set()
1111 editedFiles = set()
1112 pureRenameCopy = set()
1113 filesToChangeExecBit = {}
1115 for line in diff:
1116 diff = parseDiffTreeEntry(line)
1117 modifier = diff['status']
1118 path = diff['src']
1119 if modifier == "M":
1120 p4_edit(path)
1121 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1122 filesToChangeExecBit[path] = diff['dst_mode']
1123 editedFiles.add(path)
1124 elif modifier == "A":
1125 filesToAdd.add(path)
1126 filesToChangeExecBit[path] = diff['dst_mode']
1127 if path in filesToDelete:
1128 filesToDelete.remove(path)
1129 elif modifier == "D":
1130 filesToDelete.add(path)
1131 if path in filesToAdd:
1132 filesToAdd.remove(path)
1133 elif modifier == "C":
1134 src, dest = diff['src'], diff['dst']
1135 p4_integrate(src, dest)
1136 pureRenameCopy.add(dest)
1137 if diff['src_sha1'] != diff['dst_sha1']:
1138 p4_edit(dest)
1139 pureRenameCopy.discard(dest)
1140 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1141 p4_edit(dest)
1142 pureRenameCopy.discard(dest)
1143 filesToChangeExecBit[dest] = diff['dst_mode']
1144 os.unlink(dest)
1145 editedFiles.add(dest)
1146 elif modifier == "R":
1147 src, dest = diff['src'], diff['dst']
1148 if self.p4HasMoveCommand:
1149 p4_edit(src) # src must be open before move
1150 p4_move(src, dest) # opens for (move/delete, move/add)
1151 else:
1152 p4_integrate(src, dest)
1153 if diff['src_sha1'] != diff['dst_sha1']:
1154 p4_edit(dest)
1155 else:
1156 pureRenameCopy.add(dest)
1157 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1158 if not self.p4HasMoveCommand:
1159 p4_edit(dest) # with move: already open, writable
1160 filesToChangeExecBit[dest] = diff['dst_mode']
1161 if not self.p4HasMoveCommand:
1162 os.unlink(dest)
1163 filesToDelete.add(src)
1164 editedFiles.add(dest)
1165 else:
1166 die("unknown modifier %s for %s" % (modifier, path))
1168 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1169 patchcmd = diffcmd + " | git apply "
1170 tryPatchCmd = patchcmd + "--check -"
1171 applyPatchCmd = patchcmd + "--check --apply -"
1172 patch_succeeded = True
1174 if os.system(tryPatchCmd) != 0:
1175 fixed_rcs_keywords = False
1176 patch_succeeded = False
1177 print "Unfortunately applying the change failed!"
1179 # Patch failed, maybe it's just RCS keyword woes. Look through
1180 # the patch to see if that's possible.
1181 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1182 file = None
1183 pattern = None
1184 kwfiles = {}
1185 for file in editedFiles | filesToDelete:
1186 # did this file's delta contain RCS keywords?
1187 pattern = p4_keywords_regexp_for_file(file)
1189 if pattern:
1190 # this file is a possibility...look for RCS keywords.
1191 regexp = re.compile(pattern, re.VERBOSE)
1192 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1193 if regexp.search(line):
1194 if verbose:
1195 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1196 kwfiles[file] = pattern
1197 break
1199 for file in kwfiles:
1200 if verbose:
1201 print "zapping %s with %s" % (line,pattern)
1202 self.patchRCSKeywords(file, kwfiles[file])
1203 fixed_rcs_keywords = True
1205 if fixed_rcs_keywords:
1206 print "Retrying the patch with RCS keywords cleaned up"
1207 if os.system(tryPatchCmd) == 0:
1208 patch_succeeded = True
1210 if not patch_succeeded:
1211 for f in editedFiles:
1212 p4_revert(f)
1213 return False
1216 # Apply the patch for real, and do add/delete/+x handling.
1218 system(applyPatchCmd)
1220 for f in filesToAdd:
1221 p4_add(f)
1222 for f in filesToDelete:
1223 p4_revert(f)
1224 p4_delete(f)
1226 # Set/clear executable bits
1227 for f in filesToChangeExecBit.keys():
1228 mode = filesToChangeExecBit[f]
1229 setP4ExecBit(f, mode)
1232 # Build p4 change description, starting with the contents
1233 # of the git commit message.
1235 logMessage = extractLogMessageFromGitCommit(id)
1236 logMessage = logMessage.strip()
1237 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1239 template = self.prepareSubmitTemplate()
1240 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1242 if self.preserveUser:
1243 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1245 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1246 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1247 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1248 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1250 separatorLine = "######## everything below this line is just the diff #######\n"
1252 # diff
1253 if os.environ.has_key("P4DIFF"):
1254 del(os.environ["P4DIFF"])
1255 diff = ""
1256 for editedFile in editedFiles:
1257 diff += p4_read_pipe(['diff', '-du',
1258 wildcard_encode(editedFile)])
1260 # new file diff
1261 newdiff = ""
1262 for newFile in filesToAdd:
1263 newdiff += "==== new file ====\n"
1264 newdiff += "--- /dev/null\n"
1265 newdiff += "+++ %s\n" % newFile
1266 f = open(newFile, "r")
1267 for line in f.readlines():
1268 newdiff += "+" + line
1269 f.close()
1271 # change description file: submitTemplate, separatorLine, diff, newdiff
1272 (handle, fileName) = tempfile.mkstemp()
1273 tmpFile = os.fdopen(handle, "w+")
1274 if self.isWindows:
1275 submitTemplate = submitTemplate.replace("\n", "\r\n")
1276 separatorLine = separatorLine.replace("\n", "\r\n")
1277 newdiff = newdiff.replace("\n", "\r\n")
1278 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1279 tmpFile.close()
1281 if self.prepare_p4_only:
1283 # Leave the p4 tree prepared, and the submit template around
1284 # and let the user decide what to do next
1286 print
1287 print "P4 workspace prepared for submission."
1288 print "To submit or revert, go to client workspace"
1289 print " " + self.clientPath
1290 print
1291 print "To submit, use \"p4 submit\" to write a new description,"
1292 print "or \"p4 submit -i %s\" to use the one prepared by" \
1293 " \"git p4\"." % fileName
1294 print "You can delete the file \"%s\" when finished." % fileName
1296 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1297 print "To preserve change ownership by user %s, you must\n" \
1298 "do \"p4 change -f <change>\" after submitting and\n" \
1299 "edit the User field."
1300 if pureRenameCopy:
1301 print "After submitting, renamed files must be re-synced."
1302 print "Invoke \"p4 sync -f\" on each of these files:"
1303 for f in pureRenameCopy:
1304 print " " + f
1306 print
1307 print "To revert the changes, use \"p4 revert ...\", and delete"
1308 print "the submit template file \"%s\"" % fileName
1309 if filesToAdd:
1310 print "Since the commit adds new files, they must be deleted:"
1311 for f in filesToAdd:
1312 print " " + f
1313 print
1314 return True
1317 # Let the user edit the change description, then submit it.
1319 if self.edit_template(fileName):
1320 # read the edited message and submit
1321 ret = True
1322 tmpFile = open(fileName, "rb")
1323 message = tmpFile.read()
1324 tmpFile.close()
1325 submitTemplate = message[:message.index(separatorLine)]
1326 if self.isWindows:
1327 submitTemplate = submitTemplate.replace("\r\n", "\n")
1328 p4_write_pipe(['submit', '-i'], submitTemplate)
1330 if self.preserveUser:
1331 if p4User:
1332 # Get last changelist number. Cannot easily get it from
1333 # the submit command output as the output is
1334 # unmarshalled.
1335 changelist = self.lastP4Changelist()
1336 self.modifyChangelistUser(changelist, p4User)
1338 # The rename/copy happened by applying a patch that created a
1339 # new file. This leaves it writable, which confuses p4.
1340 for f in pureRenameCopy:
1341 p4_sync(f, "-f")
1343 else:
1344 # skip this patch
1345 ret = False
1346 print "Submission cancelled, undoing p4 changes."
1347 for f in editedFiles:
1348 p4_revert(f)
1349 for f in filesToAdd:
1350 p4_revert(f)
1351 os.remove(f)
1352 for f in filesToDelete:
1353 p4_revert(f)
1355 os.remove(fileName)
1356 return ret
1358 # Export git tags as p4 labels. Create a p4 label and then tag
1359 # with that.
1360 def exportGitTags(self, gitTags):
1361 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1362 if len(validLabelRegexp) == 0:
1363 validLabelRegexp = defaultLabelRegexp
1364 m = re.compile(validLabelRegexp)
1366 for name in gitTags:
1368 if not m.match(name):
1369 if verbose:
1370 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1371 continue
1373 # Get the p4 commit this corresponds to
1374 logMessage = extractLogMessageFromGitCommit(name)
1375 values = extractSettingsGitLog(logMessage)
1377 if not values.has_key('change'):
1378 # a tag pointing to something not sent to p4; ignore
1379 if verbose:
1380 print "git tag %s does not give a p4 commit" % name
1381 continue
1382 else:
1383 changelist = values['change']
1385 # Get the tag details.
1386 inHeader = True
1387 isAnnotated = False
1388 body = []
1389 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1390 l = l.strip()
1391 if inHeader:
1392 if re.match(r'tag\s+', l):
1393 isAnnotated = True
1394 elif re.match(r'\s*$', l):
1395 inHeader = False
1396 continue
1397 else:
1398 body.append(l)
1400 if not isAnnotated:
1401 body = ["lightweight tag imported by git p4\n"]
1403 # Create the label - use the same view as the client spec we are using
1404 clientSpec = getClientSpec()
1406 labelTemplate = "Label: %s\n" % name
1407 labelTemplate += "Description:\n"
1408 for b in body:
1409 labelTemplate += "\t" + b + "\n"
1410 labelTemplate += "View:\n"
1411 for mapping in clientSpec.mappings:
1412 labelTemplate += "\t%s\n" % mapping.depot_side.path
1414 if self.dry_run:
1415 print "Would create p4 label %s for tag" % name
1416 elif self.prepare_p4_only:
1417 print "Not creating p4 label %s for tag due to option" \
1418 " --prepare-p4-only" % name
1419 else:
1420 p4_write_pipe(["label", "-i"], labelTemplate)
1422 # Use the label
1423 p4_system(["tag", "-l", name] +
1424 ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1426 if verbose:
1427 print "created p4 label for tag %s" % name
1429 def run(self, args):
1430 if len(args) == 0:
1431 self.master = currentGitBranch()
1432 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1433 die("Detecting current git branch failed!")
1434 elif len(args) == 1:
1435 self.master = args[0]
1436 if not branchExists(self.master):
1437 die("Branch %s does not exist" % self.master)
1438 else:
1439 return False
1441 allowSubmit = gitConfig("git-p4.allowSubmit")
1442 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1443 die("%s is not in git-p4.allowSubmit" % self.master)
1445 [upstream, settings] = findUpstreamBranchPoint()
1446 self.depotPath = settings['depot-paths'][0]
1447 if len(self.origin) == 0:
1448 self.origin = upstream
1450 if self.preserveUser:
1451 if not self.canChangeChangelists():
1452 die("Cannot preserve user names without p4 super-user or admin permissions")
1454 # if not set from the command line, try the config file
1455 if self.conflict_behavior is None:
1456 val = gitConfig("git-p4.conflict")
1457 if val:
1458 if val not in self.conflict_behavior_choices:
1459 die("Invalid value '%s' for config git-p4.conflict" % val)
1460 else:
1461 val = "ask"
1462 self.conflict_behavior = val
1464 if self.verbose:
1465 print "Origin branch is " + self.origin
1467 if len(self.depotPath) == 0:
1468 print "Internal error: cannot locate perforce depot path from existing branches"
1469 sys.exit(128)
1471 self.useClientSpec = False
1472 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1473 self.useClientSpec = True
1474 if self.useClientSpec:
1475 self.clientSpecDirs = getClientSpec()
1477 if self.useClientSpec:
1478 # all files are relative to the client spec
1479 self.clientPath = getClientRoot()
1480 else:
1481 self.clientPath = p4Where(self.depotPath)
1483 if self.clientPath == "":
1484 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1486 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1487 self.oldWorkingDirectory = os.getcwd()
1489 # ensure the clientPath exists
1490 new_client_dir = False
1491 if not os.path.exists(self.clientPath):
1492 new_client_dir = True
1493 os.makedirs(self.clientPath)
1495 chdir(self.clientPath)
1496 if self.dry_run:
1497 print "Would synchronize p4 checkout in %s" % self.clientPath
1498 else:
1499 print "Synchronizing p4 checkout..."
1500 if new_client_dir:
1501 # old one was destroyed, and maybe nobody told p4
1502 p4_sync("...", "-f")
1503 else:
1504 p4_sync("...")
1505 self.check()
1507 commits = []
1508 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1509 commits.append(line.strip())
1510 commits.reverse()
1512 if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1513 self.checkAuthorship = False
1514 else:
1515 self.checkAuthorship = True
1517 if self.preserveUser:
1518 self.checkValidP4Users(commits)
1521 # Build up a set of options to be passed to diff when
1522 # submitting each commit to p4.
1524 if self.detectRenames:
1525 # command-line -M arg
1526 self.diffOpts = "-M"
1527 else:
1528 # If not explicitly set check the config variable
1529 detectRenames = gitConfig("git-p4.detectRenames")
1531 if detectRenames.lower() == "false" or detectRenames == "":
1532 self.diffOpts = ""
1533 elif detectRenames.lower() == "true":
1534 self.diffOpts = "-M"
1535 else:
1536 self.diffOpts = "-M%s" % detectRenames
1538 # no command-line arg for -C or --find-copies-harder, just
1539 # config variables
1540 detectCopies = gitConfig("git-p4.detectCopies")
1541 if detectCopies.lower() == "false" or detectCopies == "":
1542 pass
1543 elif detectCopies.lower() == "true":
1544 self.diffOpts += " -C"
1545 else:
1546 self.diffOpts += " -C%s" % detectCopies
1548 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1549 self.diffOpts += " --find-copies-harder"
1552 # Apply the commits, one at a time. On failure, ask if should
1553 # continue to try the rest of the patches, or quit.
1555 if self.dry_run:
1556 print "Would apply"
1557 applied = []
1558 last = len(commits) - 1
1559 for i, commit in enumerate(commits):
1560 if self.dry_run:
1561 print " ", read_pipe(["git", "show", "-s",
1562 "--format=format:%h %s", commit])
1563 ok = True
1564 else:
1565 ok = self.applyCommit(commit)
1566 if ok:
1567 applied.append(commit)
1568 else:
1569 if self.prepare_p4_only and i < last:
1570 print "Processing only the first commit due to option" \
1571 " --prepare-p4-only"
1572 break
1573 if i < last:
1574 quit = False
1575 while True:
1576 # prompt for what to do, or use the option/variable
1577 if self.conflict_behavior == "ask":
1578 print "What do you want to do?"
1579 response = raw_input("[s]kip this commit but apply"
1580 " the rest, or [q]uit? ")
1581 if not response:
1582 continue
1583 elif self.conflict_behavior == "skip":
1584 response = "s"
1585 elif self.conflict_behavior == "quit":
1586 response = "q"
1587 else:
1588 die("Unknown conflict_behavior '%s'" %
1589 self.conflict_behavior)
1591 if response[0] == "s":
1592 print "Skipping this commit, but applying the rest"
1593 break
1594 if response[0] == "q":
1595 print "Quitting"
1596 quit = True
1597 break
1598 if quit:
1599 break
1601 chdir(self.oldWorkingDirectory)
1603 if self.dry_run:
1604 pass
1605 elif self.prepare_p4_only:
1606 pass
1607 elif len(commits) == len(applied):
1608 print "All commits applied!"
1610 sync = P4Sync()
1611 sync.run([])
1613 rebase = P4Rebase()
1614 rebase.rebase()
1616 else:
1617 if len(applied) == 0:
1618 print "No commits applied."
1619 else:
1620 print "Applied only the commits marked with '*':"
1621 for c in commits:
1622 if c in applied:
1623 star = "*"
1624 else:
1625 star = " "
1626 print star, read_pipe(["git", "show", "-s",
1627 "--format=format:%h %s", c])
1628 print "You will have to do 'git p4 sync' and rebase."
1630 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1631 self.exportLabels = True
1633 if self.exportLabels:
1634 p4Labels = getP4Labels(self.depotPath)
1635 gitTags = getGitTags()
1637 missingGitTags = gitTags - p4Labels
1638 self.exportGitTags(missingGitTags)
1640 # exit with error unless everything applied perfecly
1641 if len(commits) != len(applied):
1642 sys.exit(1)
1644 return True
1646 class View(object):
1647 """Represent a p4 view ("p4 help views"), and map files in a
1648 repo according to the view."""
1650 class Path(object):
1651 """A depot or client path, possibly containing wildcards.
1652 The only one supported is ... at the end, currently.
1653 Initialize with the full path, with //depot or //client."""
1655 def __init__(self, path, is_depot):
1656 self.path = path
1657 self.is_depot = is_depot
1658 self.find_wildcards()
1659 # remember the prefix bit, useful for relative mappings
1660 m = re.match("(//[^/]+/)", self.path)
1661 if not m:
1662 die("Path %s does not start with //prefix/" % self.path)
1663 prefix = m.group(1)
1664 if not self.is_depot:
1665 # strip //client/ on client paths
1666 self.path = self.path[len(prefix):]
1668 def find_wildcards(self):
1669 """Make sure wildcards are valid, and set up internal
1670 variables."""
1672 self.ends_triple_dot = False
1673 # There are three wildcards allowed in p4 views
1674 # (see "p4 help views"). This code knows how to
1675 # handle "..." (only at the end), but cannot deal with
1676 # "%%n" or "*". Only check the depot_side, as p4 should
1677 # validate that the client_side matches too.
1678 if re.search(r'%%[1-9]', self.path):
1679 die("Can't handle %%n wildcards in view: %s" % self.path)
1680 if self.path.find("*") >= 0:
1681 die("Can't handle * wildcards in view: %s" % self.path)
1682 triple_dot_index = self.path.find("...")
1683 if triple_dot_index >= 0:
1684 if triple_dot_index != len(self.path) - 3:
1685 die("Can handle only single ... wildcard, at end: %s" %
1686 self.path)
1687 self.ends_triple_dot = True
1689 def ensure_compatible(self, other_path):
1690 """Make sure the wildcards agree."""
1691 if self.ends_triple_dot != other_path.ends_triple_dot:
1692 die("Both paths must end with ... if either does;\n" +
1693 "paths: %s %s" % (self.path, other_path.path))
1695 def match_wildcards(self, test_path):
1696 """See if this test_path matches us, and fill in the value
1697 of the wildcards if so. Returns a tuple of
1698 (True|False, wildcards[]). For now, only the ... at end
1699 is supported, so at most one wildcard."""
1700 if self.ends_triple_dot:
1701 dotless = self.path[:-3]
1702 if test_path.startswith(dotless):
1703 wildcard = test_path[len(dotless):]
1704 return (True, [ wildcard ])
1705 else:
1706 if test_path == self.path:
1707 return (True, [])
1708 return (False, [])
1710 def match(self, test_path):
1711 """Just return if it matches; don't bother with the wildcards."""
1712 b, _ = self.match_wildcards(test_path)
1713 return b
1715 def fill_in_wildcards(self, wildcards):
1716 """Return the relative path, with the wildcards filled in
1717 if there are any."""
1718 if self.ends_triple_dot:
1719 return self.path[:-3] + wildcards[0]
1720 else:
1721 return self.path
1723 class Mapping(object):
1724 def __init__(self, depot_side, client_side, overlay, exclude):
1725 # depot_side is without the trailing /... if it had one
1726 self.depot_side = View.Path(depot_side, is_depot=True)
1727 self.client_side = View.Path(client_side, is_depot=False)
1728 self.overlay = overlay # started with "+"
1729 self.exclude = exclude # started with "-"
1730 assert not (self.overlay and self.exclude)
1731 self.depot_side.ensure_compatible(self.client_side)
1733 def __str__(self):
1734 c = " "
1735 if self.overlay:
1736 c = "+"
1737 if self.exclude:
1738 c = "-"
1739 return "View.Mapping: %s%s -> %s" % \
1740 (c, self.depot_side.path, self.client_side.path)
1742 def map_depot_to_client(self, depot_path):
1743 """Calculate the client path if using this mapping on the
1744 given depot path; does not consider the effect of other
1745 mappings in a view. Even excluded mappings are returned."""
1746 matches, wildcards = self.depot_side.match_wildcards(depot_path)
1747 if not matches:
1748 return ""
1749 client_path = self.client_side.fill_in_wildcards(wildcards)
1750 return client_path
1753 # View methods
1755 def __init__(self):
1756 self.mappings = []
1758 def append(self, view_line):
1759 """Parse a view line, splitting it into depot and client
1760 sides. Append to self.mappings, preserving order."""
1762 # Split the view line into exactly two words. P4 enforces
1763 # structure on these lines that simplifies this quite a bit.
1765 # Either or both words may be double-quoted.
1766 # Single quotes do not matter.
1767 # Double-quote marks cannot occur inside the words.
1768 # A + or - prefix is also inside the quotes.
1769 # There are no quotes unless they contain a space.
1770 # The line is already white-space stripped.
1771 # The two words are separated by a single space.
1773 if view_line[0] == '"':
1774 # First word is double quoted. Find its end.
1775 close_quote_index = view_line.find('"', 1)
1776 if close_quote_index <= 0:
1777 die("No first-word closing quote found: %s" % view_line)
1778 depot_side = view_line[1:close_quote_index]
1779 # skip closing quote and space
1780 rhs_index = close_quote_index + 1 + 1
1781 else:
1782 space_index = view_line.find(" ")
1783 if space_index <= 0:
1784 die("No word-splitting space found: %s" % view_line)
1785 depot_side = view_line[0:space_index]
1786 rhs_index = space_index + 1
1788 if view_line[rhs_index] == '"':
1789 # Second word is double quoted. Make sure there is a
1790 # double quote at the end too.
1791 if not view_line.endswith('"'):
1792 die("View line with rhs quote should end with one: %s" %
1793 view_line)
1794 # skip the quotes
1795 client_side = view_line[rhs_index+1:-1]
1796 else:
1797 client_side = view_line[rhs_index:]
1799 # prefix + means overlay on previous mapping
1800 overlay = False
1801 if depot_side.startswith("+"):
1802 overlay = True
1803 depot_side = depot_side[1:]
1805 # prefix - means exclude this path
1806 exclude = False
1807 if depot_side.startswith("-"):
1808 exclude = True
1809 depot_side = depot_side[1:]
1811 m = View.Mapping(depot_side, client_side, overlay, exclude)
1812 self.mappings.append(m)
1814 def map_in_client(self, depot_path):
1815 """Return the relative location in the client where this
1816 depot file should live. Returns "" if the file should
1817 not be mapped in the client."""
1819 paths_filled = []
1820 client_path = ""
1822 # look at later entries first
1823 for m in self.mappings[::-1]:
1825 # see where will this path end up in the client
1826 p = m.map_depot_to_client(depot_path)
1828 if p == "":
1829 # Depot path does not belong in client. Must remember
1830 # this, as previous items should not cause files to
1831 # exist in this path either. Remember that the list is
1832 # being walked from the end, which has higher precedence.
1833 # Overlap mappings do not exclude previous mappings.
1834 if not m.overlay:
1835 paths_filled.append(m.client_side)
1837 else:
1838 # This mapping matched; no need to search any further.
1839 # But, the mapping could be rejected if the client path
1840 # has already been claimed by an earlier mapping (i.e.
1841 # one later in the list, which we are walking backwards).
1842 already_mapped_in_client = False
1843 for f in paths_filled:
1844 # this is View.Path.match
1845 if f.match(p):
1846 already_mapped_in_client = True
1847 break
1848 if not already_mapped_in_client:
1849 # Include this file, unless it is from a line that
1850 # explicitly said to exclude it.
1851 if not m.exclude:
1852 client_path = p
1854 # a match, even if rejected, always stops the search
1855 break
1857 return client_path
1859 class P4Sync(Command, P4UserMap):
1860 delete_actions = ( "delete", "move/delete", "purge" )
1862 def __init__(self):
1863 Command.__init__(self)
1864 P4UserMap.__init__(self)
1865 self.options = [
1866 optparse.make_option("--branch", dest="branch"),
1867 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1868 optparse.make_option("--changesfile", dest="changesFile"),
1869 optparse.make_option("--silent", dest="silent", action="store_true"),
1870 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1871 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1872 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1873 help="Import into refs/heads/ , not refs/remotes"),
1874 optparse.make_option("--max-changes", dest="maxChanges"),
1875 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1876 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1877 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1878 help="Only sync files that are included in the Perforce Client Spec")
1880 self.description = """Imports from Perforce into a git repository.\n
1881 example:
1882 //depot/my/project/ -- to import the current head
1883 //depot/my/project/@all -- to import everything
1884 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1886 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1888 self.usage += " //depot/path[@revRange]"
1889 self.silent = False
1890 self.createdBranches = set()
1891 self.committedChanges = set()
1892 self.branch = ""
1893 self.detectBranches = False
1894 self.detectLabels = False
1895 self.importLabels = False
1896 self.changesFile = ""
1897 self.syncWithOrigin = True
1898 self.importIntoRemotes = True
1899 self.maxChanges = ""
1900 self.isWindows = (platform.system() == "Windows")
1901 self.keepRepoPath = False
1902 self.depotPaths = None
1903 self.p4BranchesInGit = []
1904 self.cloneExclude = []
1905 self.useClientSpec = False
1906 self.useClientSpec_from_options = False
1907 self.clientSpecDirs = None
1908 self.tempBranches = []
1909 self.tempBranchLocation = "git-p4-tmp"
1911 if gitConfig("git-p4.syncFromOrigin") == "false":
1912 self.syncWithOrigin = False
1914 # Force a checkpoint in fast-import and wait for it to finish
1915 def checkpoint(self):
1916 self.gitStream.write("checkpoint\n\n")
1917 self.gitStream.write("progress checkpoint\n\n")
1918 out = self.gitOutput.readline()
1919 if self.verbose:
1920 print "checkpoint finished: " + out
1922 def extractFilesFromCommit(self, commit):
1923 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1924 for path in self.cloneExclude]
1925 files = []
1926 fnum = 0
1927 while commit.has_key("depotFile%s" % fnum):
1928 path = commit["depotFile%s" % fnum]
1930 if [p for p in self.cloneExclude
1931 if p4PathStartsWith(path, p)]:
1932 found = False
1933 else:
1934 found = [p for p in self.depotPaths
1935 if p4PathStartsWith(path, p)]
1936 if not found:
1937 fnum = fnum + 1
1938 continue
1940 file = {}
1941 file["path"] = path
1942 file["rev"] = commit["rev%s" % fnum]
1943 file["action"] = commit["action%s" % fnum]
1944 file["type"] = commit["type%s" % fnum]
1945 files.append(file)
1946 fnum = fnum + 1
1947 return files
1949 def stripRepoPath(self, path, prefixes):
1950 """When streaming files, this is called to map a p4 depot path
1951 to where it should go in git. The prefixes are either
1952 self.depotPaths, or self.branchPrefixes in the case of
1953 branch detection."""
1955 if self.useClientSpec:
1956 # branch detection moves files up a level (the branch name)
1957 # from what client spec interpretation gives
1958 path = self.clientSpecDirs.map_in_client(path)
1959 if self.detectBranches:
1960 for b in self.knownBranches:
1961 if path.startswith(b + "/"):
1962 path = path[len(b)+1:]
1964 elif self.keepRepoPath:
1965 # Preserve everything in relative path name except leading
1966 # //depot/; just look at first prefix as they all should
1967 # be in the same depot.
1968 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
1969 if p4PathStartsWith(path, depot):
1970 path = path[len(depot):]
1972 else:
1973 for p in prefixes:
1974 if p4PathStartsWith(path, p):
1975 path = path[len(p):]
1976 break
1978 path = wildcard_decode(path)
1979 return path
1981 def splitFilesIntoBranches(self, commit):
1982 """Look at each depotFile in the commit to figure out to what
1983 branch it belongs."""
1985 branches = {}
1986 fnum = 0
1987 while commit.has_key("depotFile%s" % fnum):
1988 path = commit["depotFile%s" % fnum]
1989 found = [p for p in self.depotPaths
1990 if p4PathStartsWith(path, p)]
1991 if not found:
1992 fnum = fnum + 1
1993 continue
1995 file = {}
1996 file["path"] = path
1997 file["rev"] = commit["rev%s" % fnum]
1998 file["action"] = commit["action%s" % fnum]
1999 file["type"] = commit["type%s" % fnum]
2000 fnum = fnum + 1
2002 # start with the full relative path where this file would
2003 # go in a p4 client
2004 if self.useClientSpec:
2005 relPath = self.clientSpecDirs.map_in_client(path)
2006 else:
2007 relPath = self.stripRepoPath(path, self.depotPaths)
2009 for branch in self.knownBranches.keys():
2010 # add a trailing slash so that a commit into qt/4.2foo
2011 # doesn't end up in qt/4.2, e.g.
2012 if relPath.startswith(branch + "/"):
2013 if branch not in branches:
2014 branches[branch] = []
2015 branches[branch].append(file)
2016 break
2018 return branches
2020 # output one file from the P4 stream
2021 # - helper for streamP4Files
2023 def streamOneP4File(self, file, contents):
2024 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2025 if verbose:
2026 sys.stderr.write("%s\n" % relPath)
2028 (type_base, type_mods) = split_p4_type(file["type"])
2030 git_mode = "100644"
2031 if "x" in type_mods:
2032 git_mode = "100755"
2033 if type_base == "symlink":
2034 git_mode = "120000"
2035 # p4 print on a symlink contains "target\n"; remove the newline
2036 data = ''.join(contents)
2037 contents = [data[:-1]]
2039 if type_base == "utf16":
2040 # p4 delivers different text in the python output to -G
2041 # than it does when using "print -o", or normal p4 client
2042 # operations. utf16 is converted to ascii or utf8, perhaps.
2043 # But ascii text saved as -t utf16 is completely mangled.
2044 # Invoke print -o to get the real contents.
2045 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2046 contents = [ text ]
2048 if type_base == "apple":
2049 # Apple filetype files will be streamed as a concatenation of
2050 # its appledouble header and the contents. This is useless
2051 # on both macs and non-macs. If using "print -q -o xx", it
2052 # will create "xx" with the data, and "%xx" with the header.
2053 # This is also not very useful.
2055 # Ideally, someday, this script can learn how to generate
2056 # appledouble files directly and import those to git, but
2057 # non-mac machines can never find a use for apple filetype.
2058 print "\nIgnoring apple filetype file %s" % file['depotFile']
2059 return
2061 # Perhaps windows wants unicode, utf16 newlines translated too;
2062 # but this is not doing it.
2063 if self.isWindows and type_base == "text":
2064 mangled = []
2065 for data in contents:
2066 data = data.replace("\r\n", "\n")
2067 mangled.append(data)
2068 contents = mangled
2070 # Note that we do not try to de-mangle keywords on utf16 files,
2071 # even though in theory somebody may want that.
2072 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2073 if pattern:
2074 regexp = re.compile(pattern, re.VERBOSE)
2075 text = ''.join(contents)
2076 text = regexp.sub(r'$\1$', text)
2077 contents = [ text ]
2079 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2081 # total length...
2082 length = 0
2083 for d in contents:
2084 length = length + len(d)
2086 self.gitStream.write("data %d\n" % length)
2087 for d in contents:
2088 self.gitStream.write(d)
2089 self.gitStream.write("\n")
2091 def streamOneP4Deletion(self, file):
2092 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2093 if verbose:
2094 sys.stderr.write("delete %s\n" % relPath)
2095 self.gitStream.write("D %s\n" % relPath)
2097 # handle another chunk of streaming data
2098 def streamP4FilesCb(self, marshalled):
2100 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2101 # start of a new file - output the old one first
2102 self.streamOneP4File(self.stream_file, self.stream_contents)
2103 self.stream_file = {}
2104 self.stream_contents = []
2105 self.stream_have_file_info = False
2107 # pick up the new file information... for the
2108 # 'data' field we need to append to our array
2109 for k in marshalled.keys():
2110 if k == 'data':
2111 self.stream_contents.append(marshalled['data'])
2112 else:
2113 self.stream_file[k] = marshalled[k]
2115 self.stream_have_file_info = True
2117 # Stream directly from "p4 files" into "git fast-import"
2118 def streamP4Files(self, files):
2119 filesForCommit = []
2120 filesToRead = []
2121 filesToDelete = []
2123 for f in files:
2124 # if using a client spec, only add the files that have
2125 # a path in the client
2126 if self.clientSpecDirs:
2127 if self.clientSpecDirs.map_in_client(f['path']) == "":
2128 continue
2130 filesForCommit.append(f)
2131 if f['action'] in self.delete_actions:
2132 filesToDelete.append(f)
2133 else:
2134 filesToRead.append(f)
2136 # deleted files...
2137 for f in filesToDelete:
2138 self.streamOneP4Deletion(f)
2140 if len(filesToRead) > 0:
2141 self.stream_file = {}
2142 self.stream_contents = []
2143 self.stream_have_file_info = False
2145 # curry self argument
2146 def streamP4FilesCbSelf(entry):
2147 self.streamP4FilesCb(entry)
2149 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2151 p4CmdList(["-x", "-", "print"],
2152 stdin=fileArgs,
2153 cb=streamP4FilesCbSelf)
2155 # do the last chunk
2156 if self.stream_file.has_key('depotFile'):
2157 self.streamOneP4File(self.stream_file, self.stream_contents)
2159 def make_email(self, userid):
2160 if userid in self.users:
2161 return self.users[userid]
2162 else:
2163 return "%s <a@b>" % userid
2165 # Stream a p4 tag
2166 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2167 if verbose:
2168 print "writing tag %s for commit %s" % (labelName, commit)
2169 gitStream.write("tag %s\n" % labelName)
2170 gitStream.write("from %s\n" % commit)
2172 if labelDetails.has_key('Owner'):
2173 owner = labelDetails["Owner"]
2174 else:
2175 owner = None
2177 # Try to use the owner of the p4 label, or failing that,
2178 # the current p4 user id.
2179 if owner:
2180 email = self.make_email(owner)
2181 else:
2182 email = self.make_email(self.p4UserId())
2183 tagger = "%s %s %s" % (email, epoch, self.tz)
2185 gitStream.write("tagger %s\n" % tagger)
2187 print "labelDetails=",labelDetails
2188 if labelDetails.has_key('Description'):
2189 description = labelDetails['Description']
2190 else:
2191 description = 'Label from git p4'
2193 gitStream.write("data %d\n" % len(description))
2194 gitStream.write(description)
2195 gitStream.write("\n")
2197 def commit(self, details, files, branch, parent = ""):
2198 epoch = details["time"]
2199 author = details["user"]
2201 if self.verbose:
2202 print "commit into %s" % branch
2204 # start with reading files; if that fails, we should not
2205 # create a commit.
2206 new_files = []
2207 for f in files:
2208 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2209 new_files.append (f)
2210 else:
2211 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2213 self.gitStream.write("commit %s\n" % branch)
2214 # gitStream.write("mark :%s\n" % details["change"])
2215 self.committedChanges.add(int(details["change"]))
2216 committer = ""
2217 if author not in self.users:
2218 self.getUserMapFromPerforceServer()
2219 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2221 self.gitStream.write("committer %s\n" % committer)
2223 self.gitStream.write("data <<EOT\n")
2224 self.gitStream.write(details["desc"])
2225 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2226 (','.join(self.branchPrefixes), details["change"]))
2227 if len(details['options']) > 0:
2228 self.gitStream.write(": options = %s" % details['options'])
2229 self.gitStream.write("]\nEOT\n\n")
2231 if len(parent) > 0:
2232 if self.verbose:
2233 print "parent %s" % parent
2234 self.gitStream.write("from %s\n" % parent)
2236 self.streamP4Files(new_files)
2237 self.gitStream.write("\n")
2239 change = int(details["change"])
2241 if self.labels.has_key(change):
2242 label = self.labels[change]
2243 labelDetails = label[0]
2244 labelRevisions = label[1]
2245 if self.verbose:
2246 print "Change %s is labelled %s" % (change, labelDetails)
2248 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2249 for p in self.branchPrefixes])
2251 if len(files) == len(labelRevisions):
2253 cleanedFiles = {}
2254 for info in files:
2255 if info["action"] in self.delete_actions:
2256 continue
2257 cleanedFiles[info["depotFile"]] = info["rev"]
2259 if cleanedFiles == labelRevisions:
2260 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2262 else:
2263 if not self.silent:
2264 print ("Tag %s does not match with change %s: files do not match."
2265 % (labelDetails["label"], change))
2267 else:
2268 if not self.silent:
2269 print ("Tag %s does not match with change %s: file count is different."
2270 % (labelDetails["label"], change))
2272 # Build a dictionary of changelists and labels, for "detect-labels" option.
2273 def getLabels(self):
2274 self.labels = {}
2276 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2277 if len(l) > 0 and not self.silent:
2278 print "Finding files belonging to labels in %s" % `self.depotPaths`
2280 for output in l:
2281 label = output["label"]
2282 revisions = {}
2283 newestChange = 0
2284 if self.verbose:
2285 print "Querying files for label %s" % label
2286 for file in p4CmdList(["files"] +
2287 ["%s...@%s" % (p, label)
2288 for p in self.depotPaths]):
2289 revisions[file["depotFile"]] = file["rev"]
2290 change = int(file["change"])
2291 if change > newestChange:
2292 newestChange = change
2294 self.labels[newestChange] = [output, revisions]
2296 if self.verbose:
2297 print "Label changes: %s" % self.labels.keys()
2299 # Import p4 labels as git tags. A direct mapping does not
2300 # exist, so assume that if all the files are at the same revision
2301 # then we can use that, or it's something more complicated we should
2302 # just ignore.
2303 def importP4Labels(self, stream, p4Labels):
2304 if verbose:
2305 print "import p4 labels: " + ' '.join(p4Labels)
2307 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2308 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2309 if len(validLabelRegexp) == 0:
2310 validLabelRegexp = defaultLabelRegexp
2311 m = re.compile(validLabelRegexp)
2313 for name in p4Labels:
2314 commitFound = False
2316 if not m.match(name):
2317 if verbose:
2318 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2319 continue
2321 if name in ignoredP4Labels:
2322 continue
2324 labelDetails = p4CmdList(['label', "-o", name])[0]
2326 # get the most recent changelist for each file in this label
2327 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2328 for p in self.depotPaths])
2330 if change.has_key('change'):
2331 # find the corresponding git commit; take the oldest commit
2332 changelist = int(change['change'])
2333 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2334 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2335 if len(gitCommit) == 0:
2336 print "could not find git commit for changelist %d" % changelist
2337 else:
2338 gitCommit = gitCommit.strip()
2339 commitFound = True
2340 # Convert from p4 time format
2341 try:
2342 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2343 except ValueError:
2344 print "Could not convert label time %s" % labelDetail['Update']
2345 tmwhen = 1
2347 when = int(time.mktime(tmwhen))
2348 self.streamTag(stream, name, labelDetails, gitCommit, when)
2349 if verbose:
2350 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2351 else:
2352 if verbose:
2353 print "Label %s has no changelists - possibly deleted?" % name
2355 if not commitFound:
2356 # We can't import this label; don't try again as it will get very
2357 # expensive repeatedly fetching all the files for labels that will
2358 # never be imported. If the label is moved in the future, the
2359 # ignore will need to be removed manually.
2360 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2362 def guessProjectName(self):
2363 for p in self.depotPaths:
2364 if p.endswith("/"):
2365 p = p[:-1]
2366 p = p[p.strip().rfind("/") + 1:]
2367 if not p.endswith("/"):
2368 p += "/"
2369 return p
2371 def getBranchMapping(self):
2372 lostAndFoundBranches = set()
2374 user = gitConfig("git-p4.branchUser")
2375 if len(user) > 0:
2376 command = "branches -u %s" % user
2377 else:
2378 command = "branches"
2380 for info in p4CmdList(command):
2381 details = p4Cmd(["branch", "-o", info["branch"]])
2382 viewIdx = 0
2383 while details.has_key("View%s" % viewIdx):
2384 paths = details["View%s" % viewIdx].split(" ")
2385 viewIdx = viewIdx + 1
2386 # require standard //depot/foo/... //depot/bar/... mapping
2387 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2388 continue
2389 source = paths[0]
2390 destination = paths[1]
2391 ## HACK
2392 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2393 source = source[len(self.depotPaths[0]):-4]
2394 destination = destination[len(self.depotPaths[0]):-4]
2396 if destination in self.knownBranches:
2397 if not self.silent:
2398 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2399 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2400 continue
2402 self.knownBranches[destination] = source
2404 lostAndFoundBranches.discard(destination)
2406 if source not in self.knownBranches:
2407 lostAndFoundBranches.add(source)
2409 # Perforce does not strictly require branches to be defined, so we also
2410 # check git config for a branch list.
2412 # Example of branch definition in git config file:
2413 # [git-p4]
2414 # branchList=main:branchA
2415 # branchList=main:branchB
2416 # branchList=branchA:branchC
2417 configBranches = gitConfigList("git-p4.branchList")
2418 for branch in configBranches:
2419 if branch:
2420 (source, destination) = branch.split(":")
2421 self.knownBranches[destination] = source
2423 lostAndFoundBranches.discard(destination)
2425 if source not in self.knownBranches:
2426 lostAndFoundBranches.add(source)
2429 for branch in lostAndFoundBranches:
2430 self.knownBranches[branch] = branch
2432 def getBranchMappingFromGitBranches(self):
2433 branches = p4BranchesInGit(self.importIntoRemotes)
2434 for branch in branches.keys():
2435 if branch == "master":
2436 branch = "main"
2437 else:
2438 branch = branch[len(self.projectName):]
2439 self.knownBranches[branch] = branch
2441 def listExistingP4GitBranches(self):
2442 # branches holds mapping from name to commit
2443 branches = p4BranchesInGit(self.importIntoRemotes)
2444 self.p4BranchesInGit = branches.keys()
2445 for branch in branches.keys():
2446 self.initialParents[self.refPrefix + branch] = branches[branch]
2448 def updateOptionDict(self, d):
2449 option_keys = {}
2450 if self.keepRepoPath:
2451 option_keys['keepRepoPath'] = 1
2453 d["options"] = ' '.join(sorted(option_keys.keys()))
2455 def readOptions(self, d):
2456 self.keepRepoPath = (d.has_key('options')
2457 and ('keepRepoPath' in d['options']))
2459 def gitRefForBranch(self, branch):
2460 if branch == "main":
2461 return self.refPrefix + "master"
2463 if len(branch) <= 0:
2464 return branch
2466 return self.refPrefix + self.projectName + branch
2468 def gitCommitByP4Change(self, ref, change):
2469 if self.verbose:
2470 print "looking in ref " + ref + " for change %s using bisect..." % change
2472 earliestCommit = ""
2473 latestCommit = parseRevision(ref)
2475 while True:
2476 if self.verbose:
2477 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2478 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2479 if len(next) == 0:
2480 if self.verbose:
2481 print "argh"
2482 return ""
2483 log = extractLogMessageFromGitCommit(next)
2484 settings = extractSettingsGitLog(log)
2485 currentChange = int(settings['change'])
2486 if self.verbose:
2487 print "current change %s" % currentChange
2489 if currentChange == change:
2490 if self.verbose:
2491 print "found %s" % next
2492 return next
2494 if currentChange < change:
2495 earliestCommit = "^%s" % next
2496 else:
2497 latestCommit = "%s" % next
2499 return ""
2501 def importNewBranch(self, branch, maxChange):
2502 # make fast-import flush all changes to disk and update the refs using the checkpoint
2503 # command so that we can try to find the branch parent in the git history
2504 self.gitStream.write("checkpoint\n\n");
2505 self.gitStream.flush();
2506 branchPrefix = self.depotPaths[0] + branch + "/"
2507 range = "@1,%s" % maxChange
2508 #print "prefix" + branchPrefix
2509 changes = p4ChangesForPaths([branchPrefix], range)
2510 if len(changes) <= 0:
2511 return False
2512 firstChange = changes[0]
2513 #print "first change in branch: %s" % firstChange
2514 sourceBranch = self.knownBranches[branch]
2515 sourceDepotPath = self.depotPaths[0] + sourceBranch
2516 sourceRef = self.gitRefForBranch(sourceBranch)
2517 #print "source " + sourceBranch
2519 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2520 #print "branch parent: %s" % branchParentChange
2521 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2522 if len(gitParent) > 0:
2523 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2524 #print "parent git commit: %s" % gitParent
2526 self.importChanges(changes)
2527 return True
2529 def searchParent(self, parent, branch, target):
2530 parentFound = False
2531 for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2532 blob = blob.strip()
2533 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2534 parentFound = True
2535 if self.verbose:
2536 print "Found parent of %s in commit %s" % (branch, blob)
2537 break
2538 if parentFound:
2539 return blob
2540 else:
2541 return None
2543 def importChanges(self, changes):
2544 cnt = 1
2545 for change in changes:
2546 description = p4Cmd(["describe", str(change)])
2547 self.updateOptionDict(description)
2549 if not self.silent:
2550 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2551 sys.stdout.flush()
2552 cnt = cnt + 1
2554 try:
2555 if self.detectBranches:
2556 branches = self.splitFilesIntoBranches(description)
2557 for branch in branches.keys():
2558 ## HACK --hwn
2559 branchPrefix = self.depotPaths[0] + branch + "/"
2560 self.branchPrefixes = [ branchPrefix ]
2562 parent = ""
2564 filesForCommit = branches[branch]
2566 if self.verbose:
2567 print "branch is %s" % branch
2569 self.updatedBranches.add(branch)
2571 if branch not in self.createdBranches:
2572 self.createdBranches.add(branch)
2573 parent = self.knownBranches[branch]
2574 if parent == branch:
2575 parent = ""
2576 else:
2577 fullBranch = self.projectName + branch
2578 if fullBranch not in self.p4BranchesInGit:
2579 if not self.silent:
2580 print("\n Importing new branch %s" % fullBranch);
2581 if self.importNewBranch(branch, change - 1):
2582 parent = ""
2583 self.p4BranchesInGit.append(fullBranch)
2584 if not self.silent:
2585 print("\n Resuming with change %s" % change);
2587 if self.verbose:
2588 print "parent determined through known branches: %s" % parent
2590 branch = self.gitRefForBranch(branch)
2591 parent = self.gitRefForBranch(parent)
2593 if self.verbose:
2594 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2596 if len(parent) == 0 and branch in self.initialParents:
2597 parent = self.initialParents[branch]
2598 del self.initialParents[branch]
2600 blob = None
2601 if len(parent) > 0:
2602 tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2603 if self.verbose:
2604 print "Creating temporary branch: " + tempBranch
2605 self.commit(description, filesForCommit, tempBranch)
2606 self.tempBranches.append(tempBranch)
2607 self.checkpoint()
2608 blob = self.searchParent(parent, branch, tempBranch)
2609 if blob:
2610 self.commit(description, filesForCommit, branch, blob)
2611 else:
2612 if self.verbose:
2613 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2614 self.commit(description, filesForCommit, branch, parent)
2615 else:
2616 files = self.extractFilesFromCommit(description)
2617 self.commit(description, files, self.branch,
2618 self.initialParent)
2619 self.initialParent = ""
2620 except IOError:
2621 print self.gitError.read()
2622 sys.exit(1)
2624 def importHeadRevision(self, revision):
2625 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2627 details = {}
2628 details["user"] = "git perforce import user"
2629 details["desc"] = ("Initial import of %s from the state at revision %s\n"
2630 % (' '.join(self.depotPaths), revision))
2631 details["change"] = revision
2632 newestRevision = 0
2634 fileCnt = 0
2635 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2637 for info in p4CmdList(["files"] + fileArgs):
2639 if 'code' in info and info['code'] == 'error':
2640 sys.stderr.write("p4 returned an error: %s\n"
2641 % info['data'])
2642 if info['data'].find("must refer to client") >= 0:
2643 sys.stderr.write("This particular p4 error is misleading.\n")
2644 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2645 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
2646 sys.exit(1)
2647 if 'p4ExitCode' in info:
2648 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2649 sys.exit(1)
2652 change = int(info["change"])
2653 if change > newestRevision:
2654 newestRevision = change
2656 if info["action"] in self.delete_actions:
2657 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2658 #fileCnt = fileCnt + 1
2659 continue
2661 for prop in ["depotFile", "rev", "action", "type" ]:
2662 details["%s%s" % (prop, fileCnt)] = info[prop]
2664 fileCnt = fileCnt + 1
2666 details["change"] = newestRevision
2668 # Use time from top-most change so that all git p4 clones of
2669 # the same p4 repo have the same commit SHA1s.
2670 res = p4CmdList("describe -s %d" % newestRevision)
2671 newestTime = None
2672 for r in res:
2673 if r.has_key('time'):
2674 newestTime = int(r['time'])
2675 if newestTime is None:
2676 die("\"describe -s\" on newest change %d did not give a time")
2677 details["time"] = newestTime
2679 self.updateOptionDict(details)
2680 try:
2681 self.commit(details, self.extractFilesFromCommit(details), self.branch)
2682 except IOError:
2683 print "IO error with git fast-import. Is your git version recent enough?"
2684 print self.gitError.read()
2687 def run(self, args):
2688 self.depotPaths = []
2689 self.changeRange = ""
2690 self.initialParent = ""
2691 self.previousDepotPaths = []
2693 # map from branch depot path to parent branch
2694 self.knownBranches = {}
2695 self.initialParents = {}
2696 self.hasOrigin = originP4BranchesExist()
2697 if not self.syncWithOrigin:
2698 self.hasOrigin = False
2700 if self.importIntoRemotes:
2701 self.refPrefix = "refs/remotes/p4/"
2702 else:
2703 self.refPrefix = "refs/heads/p4/"
2705 if self.syncWithOrigin and self.hasOrigin:
2706 if not self.silent:
2707 print "Syncing with origin first by calling git fetch origin"
2708 system("git fetch origin")
2710 if len(self.branch) == 0:
2711 self.branch = self.refPrefix + "master"
2712 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2713 system("git update-ref %s refs/heads/p4" % self.branch)
2714 system("git branch -D p4");
2715 # create it /after/ importing, when master exists
2716 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2717 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2719 # accept either the command-line option, or the configuration variable
2720 if self.useClientSpec:
2721 # will use this after clone to set the variable
2722 self.useClientSpec_from_options = True
2723 else:
2724 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2725 self.useClientSpec = True
2726 if self.useClientSpec:
2727 self.clientSpecDirs = getClientSpec()
2729 # TODO: should always look at previous commits,
2730 # merge with previous imports, if possible.
2731 if args == []:
2732 if self.hasOrigin:
2733 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2734 self.listExistingP4GitBranches()
2736 if len(self.p4BranchesInGit) > 1:
2737 if not self.silent:
2738 print "Importing from/into multiple branches"
2739 self.detectBranches = True
2741 if self.verbose:
2742 print "branches: %s" % self.p4BranchesInGit
2744 p4Change = 0
2745 for branch in self.p4BranchesInGit:
2746 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
2748 settings = extractSettingsGitLog(logMsg)
2750 self.readOptions(settings)
2751 if (settings.has_key('depot-paths')
2752 and settings.has_key ('change')):
2753 change = int(settings['change']) + 1
2754 p4Change = max(p4Change, change)
2756 depotPaths = sorted(settings['depot-paths'])
2757 if self.previousDepotPaths == []:
2758 self.previousDepotPaths = depotPaths
2759 else:
2760 paths = []
2761 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2762 prev_list = prev.split("/")
2763 cur_list = cur.split("/")
2764 for i in range(0, min(len(cur_list), len(prev_list))):
2765 if cur_list[i] <> prev_list[i]:
2766 i = i - 1
2767 break
2769 paths.append ("/".join(cur_list[:i + 1]))
2771 self.previousDepotPaths = paths
2773 if p4Change > 0:
2774 self.depotPaths = sorted(self.previousDepotPaths)
2775 self.changeRange = "@%s,#head" % p4Change
2776 if not self.detectBranches:
2777 self.initialParent = parseRevision(self.branch)
2778 if not self.silent and not self.detectBranches:
2779 print "Performing incremental import into %s git branch" % self.branch
2781 if not self.branch.startswith("refs/"):
2782 self.branch = "refs/heads/" + self.branch
2784 if len(args) == 0 and self.depotPaths:
2785 if not self.silent:
2786 print "Depot paths: %s" % ' '.join(self.depotPaths)
2787 else:
2788 if self.depotPaths and self.depotPaths != args:
2789 print ("previous import used depot path %s and now %s was specified. "
2790 "This doesn't work!" % (' '.join (self.depotPaths),
2791 ' '.join (args)))
2792 sys.exit(1)
2794 self.depotPaths = sorted(args)
2796 revision = ""
2797 self.users = {}
2799 # Make sure no revision specifiers are used when --changesfile
2800 # is specified.
2801 bad_changesfile = False
2802 if len(self.changesFile) > 0:
2803 for p in self.depotPaths:
2804 if p.find("@") >= 0 or p.find("#") >= 0:
2805 bad_changesfile = True
2806 break
2807 if bad_changesfile:
2808 die("Option --changesfile is incompatible with revision specifiers")
2810 newPaths = []
2811 for p in self.depotPaths:
2812 if p.find("@") != -1:
2813 atIdx = p.index("@")
2814 self.changeRange = p[atIdx:]
2815 if self.changeRange == "@all":
2816 self.changeRange = ""
2817 elif ',' not in self.changeRange:
2818 revision = self.changeRange
2819 self.changeRange = ""
2820 p = p[:atIdx]
2821 elif p.find("#") != -1:
2822 hashIdx = p.index("#")
2823 revision = p[hashIdx:]
2824 p = p[:hashIdx]
2825 elif self.previousDepotPaths == []:
2826 # pay attention to changesfile, if given, else import
2827 # the entire p4 tree at the head revision
2828 if len(self.changesFile) == 0:
2829 revision = "#head"
2831 p = re.sub ("\.\.\.$", "", p)
2832 if not p.endswith("/"):
2833 p += "/"
2835 newPaths.append(p)
2837 self.depotPaths = newPaths
2839 # --detect-branches may change this for each branch
2840 self.branchPrefixes = self.depotPaths
2842 self.loadUserMapFromCache()
2843 self.labels = {}
2844 if self.detectLabels:
2845 self.getLabels();
2847 if self.detectBranches:
2848 ## FIXME - what's a P4 projectName ?
2849 self.projectName = self.guessProjectName()
2851 if self.hasOrigin:
2852 self.getBranchMappingFromGitBranches()
2853 else:
2854 self.getBranchMapping()
2855 if self.verbose:
2856 print "p4-git branches: %s" % self.p4BranchesInGit
2857 print "initial parents: %s" % self.initialParents
2858 for b in self.p4BranchesInGit:
2859 if b != "master":
2861 ## FIXME
2862 b = b[len(self.projectName):]
2863 self.createdBranches.add(b)
2865 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2867 importProcess = subprocess.Popen(["git", "fast-import"],
2868 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2869 stderr=subprocess.PIPE);
2870 self.gitOutput = importProcess.stdout
2871 self.gitStream = importProcess.stdin
2872 self.gitError = importProcess.stderr
2874 if revision:
2875 self.importHeadRevision(revision)
2876 else:
2877 changes = []
2879 if len(self.changesFile) > 0:
2880 output = open(self.changesFile).readlines()
2881 changeSet = set()
2882 for line in output:
2883 changeSet.add(int(line))
2885 for change in changeSet:
2886 changes.append(change)
2888 changes.sort()
2889 else:
2890 # catch "git p4 sync" with no new branches, in a repo that
2891 # does not have any existing p4 branches
2892 if len(args) == 0 and not self.p4BranchesInGit:
2893 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2894 if self.verbose:
2895 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2896 self.changeRange)
2897 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2899 if len(self.maxChanges) > 0:
2900 changes = changes[:min(int(self.maxChanges), len(changes))]
2902 if len(changes) == 0:
2903 if not self.silent:
2904 print "No changes to import!"
2905 else:
2906 if not self.silent and not self.detectBranches:
2907 print "Import destination: %s" % self.branch
2909 self.updatedBranches = set()
2911 self.importChanges(changes)
2913 if not self.silent:
2914 print ""
2915 if len(self.updatedBranches) > 0:
2916 sys.stdout.write("Updated branches: ")
2917 for b in self.updatedBranches:
2918 sys.stdout.write("%s " % b)
2919 sys.stdout.write("\n")
2921 if gitConfig("git-p4.importLabels", "--bool") == "true":
2922 self.importLabels = True
2924 if self.importLabels:
2925 p4Labels = getP4Labels(self.depotPaths)
2926 gitTags = getGitTags()
2928 missingP4Labels = p4Labels - gitTags
2929 self.importP4Labels(self.gitStream, missingP4Labels)
2931 self.gitStream.close()
2932 if importProcess.wait() != 0:
2933 die("fast-import failed: %s" % self.gitError.read())
2934 self.gitOutput.close()
2935 self.gitError.close()
2937 # Cleanup temporary branches created during import
2938 if self.tempBranches != []:
2939 for branch in self.tempBranches:
2940 read_pipe("git update-ref -d %s" % branch)
2941 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2943 return True
2945 class P4Rebase(Command):
2946 def __init__(self):
2947 Command.__init__(self)
2948 self.options = [
2949 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2951 self.importLabels = False
2952 self.description = ("Fetches the latest revision from perforce and "
2953 + "rebases the current work (branch) against it")
2955 def run(self, args):
2956 sync = P4Sync()
2957 sync.importLabels = self.importLabels
2958 sync.run([])
2960 return self.rebase()
2962 def rebase(self):
2963 if os.system("git update-index --refresh") != 0:
2964 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2965 if len(read_pipe("git diff-index HEAD --")) > 0:
2966 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2968 [upstream, settings] = findUpstreamBranchPoint()
2969 if len(upstream) == 0:
2970 die("Cannot find upstream branchpoint for rebase")
2972 # the branchpoint may be p4/foo~3, so strip off the parent
2973 upstream = re.sub("~[0-9]+$", "", upstream)
2975 print "Rebasing the current branch onto %s" % upstream
2976 oldHead = read_pipe("git rev-parse HEAD").strip()
2977 system("git rebase %s" % upstream)
2978 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2979 return True
2981 class P4Clone(P4Sync):
2982 def __init__(self):
2983 P4Sync.__init__(self)
2984 self.description = "Creates a new git repository and imports from Perforce into it"
2985 self.usage = "usage: %prog [options] //depot/path[@revRange]"
2986 self.options += [
2987 optparse.make_option("--destination", dest="cloneDestination",
2988 action='store', default=None,
2989 help="where to leave result of the clone"),
2990 optparse.make_option("-/", dest="cloneExclude",
2991 action="append", type="string",
2992 help="exclude depot path"),
2993 optparse.make_option("--bare", dest="cloneBare",
2994 action="store_true", default=False),
2996 self.cloneDestination = None
2997 self.needsGit = False
2998 self.cloneBare = False
3000 # This is required for the "append" cloneExclude action
3001 def ensure_value(self, attr, value):
3002 if not hasattr(self, attr) or getattr(self, attr) is None:
3003 setattr(self, attr, value)
3004 return getattr(self, attr)
3006 def defaultDestination(self, args):
3007 ## TODO: use common prefix of args?
3008 depotPath = args[0]
3009 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3010 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3011 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3012 depotDir = re.sub(r"/$", "", depotDir)
3013 return os.path.split(depotDir)[1]
3015 def run(self, args):
3016 if len(args) < 1:
3017 return False
3019 if self.keepRepoPath and not self.cloneDestination:
3020 sys.stderr.write("Must specify destination for --keep-path\n")
3021 sys.exit(1)
3023 depotPaths = args
3025 if not self.cloneDestination and len(depotPaths) > 1:
3026 self.cloneDestination = depotPaths[-1]
3027 depotPaths = depotPaths[:-1]
3029 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3030 for p in depotPaths:
3031 if not p.startswith("//"):
3032 return False
3034 if not self.cloneDestination:
3035 self.cloneDestination = self.defaultDestination(args)
3037 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3039 if not os.path.exists(self.cloneDestination):
3040 os.makedirs(self.cloneDestination)
3041 chdir(self.cloneDestination)
3043 init_cmd = [ "git", "init" ]
3044 if self.cloneBare:
3045 init_cmd.append("--bare")
3046 subprocess.check_call(init_cmd)
3048 if not P4Sync.run(self, depotPaths):
3049 return False
3050 if self.branch != "master":
3051 if self.importIntoRemotes:
3052 masterbranch = "refs/remotes/p4/master"
3053 else:
3054 masterbranch = "refs/heads/p4/master"
3055 if gitBranchExists(masterbranch):
3056 system("git branch master %s" % masterbranch)
3057 if not self.cloneBare:
3058 system("git checkout -f")
3059 else:
3060 print "Could not detect main branch. No checkout/master branch created."
3062 # auto-set this variable if invoked with --use-client-spec
3063 if self.useClientSpec_from_options:
3064 system("git config --bool git-p4.useclientspec true")
3066 return True
3068 class P4Branches(Command):
3069 def __init__(self):
3070 Command.__init__(self)
3071 self.options = [ ]
3072 self.description = ("Shows the git branches that hold imports and their "
3073 + "corresponding perforce depot paths")
3074 self.verbose = False
3076 def run(self, args):
3077 if originP4BranchesExist():
3078 createOrUpdateBranchesFromOrigin()
3080 cmdline = "git rev-parse --symbolic "
3081 cmdline += " --remotes"
3083 for line in read_pipe_lines(cmdline):
3084 line = line.strip()
3086 if not line.startswith('p4/') or line == "p4/HEAD":
3087 continue
3088 branch = line
3090 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3091 settings = extractSettingsGitLog(log)
3093 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3094 return True
3096 class HelpFormatter(optparse.IndentedHelpFormatter):
3097 def __init__(self):
3098 optparse.IndentedHelpFormatter.__init__(self)
3100 def format_description(self, description):
3101 if description:
3102 return description + "\n"
3103 else:
3104 return ""
3106 def printUsage(commands):
3107 print "usage: %s <command> [options]" % sys.argv[0]
3108 print ""
3109 print "valid commands: %s" % ", ".join(commands)
3110 print ""
3111 print "Try %s <command> --help for command specific help." % sys.argv[0]
3112 print ""
3114 commands = {
3115 "debug" : P4Debug,
3116 "submit" : P4Submit,
3117 "commit" : P4Submit,
3118 "sync" : P4Sync,
3119 "rebase" : P4Rebase,
3120 "clone" : P4Clone,
3121 "rollback" : P4RollBack,
3122 "branches" : P4Branches
3126 def main():
3127 if len(sys.argv[1:]) == 0:
3128 printUsage(commands.keys())
3129 sys.exit(2)
3131 cmd = ""
3132 cmdName = sys.argv[1]
3133 try:
3134 klass = commands[cmdName]
3135 cmd = klass()
3136 except KeyError:
3137 print "unknown command %s" % cmdName
3138 print ""
3139 printUsage(commands.keys())
3140 sys.exit(2)
3142 options = cmd.options
3143 cmd.gitdir = os.environ.get("GIT_DIR", None)
3145 args = sys.argv[2:]
3147 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3148 if cmd.needsGit:
3149 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3151 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3152 options,
3153 description = cmd.description,
3154 formatter = HelpFormatter())
3156 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3157 global verbose
3158 verbose = cmd.verbose
3159 if cmd.needsGit:
3160 if cmd.gitdir == None:
3161 cmd.gitdir = os.path.abspath(".git")
3162 if not isValidGitDir(cmd.gitdir):
3163 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3164 if os.path.exists(cmd.gitdir):
3165 cdup = read_pipe("git rev-parse --show-cdup").strip()
3166 if len(cdup) > 0:
3167 chdir(cdup);
3169 if not isValidGitDir(cmd.gitdir):
3170 if isValidGitDir(cmd.gitdir + "/.git"):
3171 cmd.gitdir += "/.git"
3172 else:
3173 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3175 os.environ["GIT_DIR"] = cmd.gitdir
3177 if not cmd.run(args):
3178 parser.print_help()
3179 sys.exit(2)
3182 if __name__ == '__main__':
3183 main()