Merge branch 'maint'
[git/jrn.git] / git-p4.py
blob69f1452cf80066a002fe58bb1eeebda7fe63ba8b
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import sys
12 if sys.hexversion < 0x02040000:
13 # The limiter is the subprocess module
14 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
15 sys.exit(1)
17 import optparse, os, marshal, subprocess, shelve
18 import tempfile, getopt, os.path, time, platform
19 import re, shutil
21 verbose = False
23 # Only labels/tags matching this will be imported/exported
24 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
26 def p4_build_cmd(cmd):
27 """Build a suitable p4 command line.
29 This consolidates building and returning a p4 command line into one
30 location. It means that hooking into the environment, or other configuration
31 can be done more easily.
32 """
33 real_cmd = ["p4"]
35 user = gitConfig("git-p4.user")
36 if len(user) > 0:
37 real_cmd += ["-u",user]
39 password = gitConfig("git-p4.password")
40 if len(password) > 0:
41 real_cmd += ["-P", password]
43 port = gitConfig("git-p4.port")
44 if len(port) > 0:
45 real_cmd += ["-p", port]
47 host = gitConfig("git-p4.host")
48 if len(host) > 0:
49 real_cmd += ["-H", host]
51 client = gitConfig("git-p4.client")
52 if len(client) > 0:
53 real_cmd += ["-c", client]
56 if isinstance(cmd,basestring):
57 real_cmd = ' '.join(real_cmd) + ' ' + cmd
58 else:
59 real_cmd += cmd
60 return real_cmd
62 def chdir(dir):
63 # P4 uses the PWD environment variable rather than getcwd(). Since we're
64 # not using the shell, we have to set it ourselves. This path could
65 # be relative, so go there first, then figure out where we ended up.
66 os.chdir(dir)
67 os.environ['PWD'] = os.getcwd()
69 def die(msg):
70 if verbose:
71 raise Exception(msg)
72 else:
73 sys.stderr.write(msg + "\n")
74 sys.exit(1)
76 def write_pipe(c, stdin):
77 if verbose:
78 sys.stderr.write('Writing pipe: %s\n' % str(c))
80 expand = isinstance(c,basestring)
81 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
82 pipe = p.stdin
83 val = pipe.write(stdin)
84 pipe.close()
85 if p.wait():
86 die('Command failed: %s' % str(c))
88 return val
90 def p4_write_pipe(c, stdin):
91 real_cmd = p4_build_cmd(c)
92 return write_pipe(real_cmd, stdin)
94 def read_pipe(c, ignore_error=False):
95 if verbose:
96 sys.stderr.write('Reading pipe: %s\n' % str(c))
98 expand = isinstance(c,basestring)
99 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
100 pipe = p.stdout
101 val = pipe.read()
102 if p.wait() and not ignore_error:
103 die('Command failed: %s' % str(c))
105 return val
107 def p4_read_pipe(c, ignore_error=False):
108 real_cmd = p4_build_cmd(c)
109 return read_pipe(real_cmd, ignore_error)
111 def read_pipe_lines(c):
112 if verbose:
113 sys.stderr.write('Reading pipe: %s\n' % str(c))
115 expand = isinstance(c, basestring)
116 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
117 pipe = p.stdout
118 val = pipe.readlines()
119 if pipe.close() or p.wait():
120 die('Command failed: %s' % str(c))
122 return val
124 def p4_read_pipe_lines(c):
125 """Specifically invoke p4 on the command supplied. """
126 real_cmd = p4_build_cmd(c)
127 return read_pipe_lines(real_cmd)
129 def p4_has_command(cmd):
130 """Ask p4 for help on this command. If it returns an error, the
131 command does not exist in this version of p4."""
132 real_cmd = p4_build_cmd(["help", cmd])
133 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
134 stderr=subprocess.PIPE)
135 p.communicate()
136 return p.returncode == 0
138 def p4_has_move_command():
139 """See if the move command exists, that it supports -k, and that
140 it has not been administratively disabled. The arguments
141 must be correct, but the filenames do not have to exist. Use
142 ones with wildcards so even if they exist, it will fail."""
144 if not p4_has_command("move"):
145 return False
146 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
147 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
148 (out, err) = p.communicate()
149 # return code will be 1 in either case
150 if err.find("Invalid option") >= 0:
151 return False
152 if err.find("disabled") >= 0:
153 return False
154 # assume it failed because @... was invalid changelist
155 return True
157 def system(cmd):
158 expand = isinstance(cmd,basestring)
159 if verbose:
160 sys.stderr.write("executing %s\n" % str(cmd))
161 subprocess.check_call(cmd, shell=expand)
163 def p4_system(cmd):
164 """Specifically invoke p4 as the system command. """
165 real_cmd = p4_build_cmd(cmd)
166 expand = isinstance(real_cmd, basestring)
167 subprocess.check_call(real_cmd, shell=expand)
169 def p4_integrate(src, dest):
170 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
172 def p4_sync(f, *options):
173 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
175 def p4_add(f):
176 # forcibly add file names with wildcards
177 if wildcard_present(f):
178 p4_system(["add", "-f", f])
179 else:
180 p4_system(["add", f])
182 def p4_delete(f):
183 p4_system(["delete", wildcard_encode(f)])
185 def p4_edit(f):
186 p4_system(["edit", wildcard_encode(f)])
188 def p4_revert(f):
189 p4_system(["revert", wildcard_encode(f)])
191 def p4_reopen(type, f):
192 p4_system(["reopen", "-t", type, wildcard_encode(f)])
194 def p4_move(src, dest):
195 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
197 def p4_describe(change):
198 """Make sure it returns a valid result by checking for
199 the presence of field "time". Return a dict of the
200 results."""
202 ds = p4CmdList(["describe", "-s", str(change)])
203 if len(ds) != 1:
204 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
206 d = ds[0]
208 if "p4ExitCode" in d:
209 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
210 str(d)))
211 if "code" in d:
212 if d["code"] == "error":
213 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
215 if "time" not in d:
216 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
218 return d
221 # Canonicalize the p4 type and return a tuple of the
222 # base type, plus any modifiers. See "p4 help filetypes"
223 # for a list and explanation.
225 def split_p4_type(p4type):
227 p4_filetypes_historical = {
228 "ctempobj": "binary+Sw",
229 "ctext": "text+C",
230 "cxtext": "text+Cx",
231 "ktext": "text+k",
232 "kxtext": "text+kx",
233 "ltext": "text+F",
234 "tempobj": "binary+FSw",
235 "ubinary": "binary+F",
236 "uresource": "resource+F",
237 "uxbinary": "binary+Fx",
238 "xbinary": "binary+x",
239 "xltext": "text+Fx",
240 "xtempobj": "binary+Swx",
241 "xtext": "text+x",
242 "xunicode": "unicode+x",
243 "xutf16": "utf16+x",
245 if p4type in p4_filetypes_historical:
246 p4type = p4_filetypes_historical[p4type]
247 mods = ""
248 s = p4type.split("+")
249 base = s[0]
250 mods = ""
251 if len(s) > 1:
252 mods = s[1]
253 return (base, mods)
256 # return the raw p4 type of a file (text, text+ko, etc)
258 def p4_type(file):
259 results = p4CmdList(["fstat", "-T", "headType", file])
260 return results[0]['headType']
263 # Given a type base and modifier, return a regexp matching
264 # the keywords that can be expanded in the file
266 def p4_keywords_regexp_for_type(base, type_mods):
267 if base in ("text", "unicode", "binary"):
268 kwords = None
269 if "ko" in type_mods:
270 kwords = 'Id|Header'
271 elif "k" in type_mods:
272 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
273 else:
274 return None
275 pattern = r"""
276 \$ # Starts with a dollar, followed by...
277 (%s) # one of the keywords, followed by...
278 (:[^$\n]+)? # possibly an old expansion, followed by...
279 \$ # another dollar
280 """ % kwords
281 return pattern
282 else:
283 return None
286 # Given a file, return a regexp matching the possible
287 # RCS keywords that will be expanded, or None for files
288 # with kw expansion turned off.
290 def p4_keywords_regexp_for_file(file):
291 if not os.path.exists(file):
292 return None
293 else:
294 (type_base, type_mods) = split_p4_type(p4_type(file))
295 return p4_keywords_regexp_for_type(type_base, type_mods)
297 def setP4ExecBit(file, mode):
298 # Reopens an already open file and changes the execute bit to match
299 # the execute bit setting in the passed in mode.
301 p4Type = "+x"
303 if not isModeExec(mode):
304 p4Type = getP4OpenedType(file)
305 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
306 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
307 if p4Type[-1] == "+":
308 p4Type = p4Type[0:-1]
310 p4_reopen(p4Type, file)
312 def getP4OpenedType(file):
313 # Returns the perforce file type for the given file.
315 result = p4_read_pipe(["opened", wildcard_encode(file)])
316 match = re.match(".*\((.+)\)\r?$", result)
317 if match:
318 return match.group(1)
319 else:
320 die("Could not determine file type for %s (result: '%s')" % (file, result))
322 # Return the set of all p4 labels
323 def getP4Labels(depotPaths):
324 labels = set()
325 if isinstance(depotPaths,basestring):
326 depotPaths = [depotPaths]
328 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
329 label = l['label']
330 labels.add(label)
332 return labels
334 # Return the set of all git tags
335 def getGitTags():
336 gitTags = set()
337 for line in read_pipe_lines(["git", "tag"]):
338 tag = line.strip()
339 gitTags.add(tag)
340 return gitTags
342 def diffTreePattern():
343 # This is a simple generator for the diff tree regex pattern. This could be
344 # a class variable if this and parseDiffTreeEntry were a part of a class.
345 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
346 while True:
347 yield pattern
349 def parseDiffTreeEntry(entry):
350 """Parses a single diff tree entry into its component elements.
352 See git-diff-tree(1) manpage for details about the format of the diff
353 output. This method returns a dictionary with the following elements:
355 src_mode - The mode of the source file
356 dst_mode - The mode of the destination file
357 src_sha1 - The sha1 for the source file
358 dst_sha1 - The sha1 fr the destination file
359 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
360 status_score - The score for the status (applicable for 'C' and 'R'
361 statuses). This is None if there is no score.
362 src - The path for the source file.
363 dst - The path for the destination file. This is only present for
364 copy or renames. If it is not present, this is None.
366 If the pattern is not matched, None is returned."""
368 match = diffTreePattern().next().match(entry)
369 if match:
370 return {
371 'src_mode': match.group(1),
372 'dst_mode': match.group(2),
373 'src_sha1': match.group(3),
374 'dst_sha1': match.group(4),
375 'status': match.group(5),
376 'status_score': match.group(6),
377 'src': match.group(7),
378 'dst': match.group(10)
380 return None
382 def isModeExec(mode):
383 # Returns True if the given git mode represents an executable file,
384 # otherwise False.
385 return mode[-3:] == "755"
387 def isModeExecChanged(src_mode, dst_mode):
388 return isModeExec(src_mode) != isModeExec(dst_mode)
390 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
392 if isinstance(cmd,basestring):
393 cmd = "-G " + cmd
394 expand = True
395 else:
396 cmd = ["-G"] + cmd
397 expand = False
399 cmd = p4_build_cmd(cmd)
400 if verbose:
401 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
403 # Use a temporary file to avoid deadlocks without
404 # subprocess.communicate(), which would put another copy
405 # of stdout into memory.
406 stdin_file = None
407 if stdin is not None:
408 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
409 if isinstance(stdin,basestring):
410 stdin_file.write(stdin)
411 else:
412 for i in stdin:
413 stdin_file.write(i + '\n')
414 stdin_file.flush()
415 stdin_file.seek(0)
417 p4 = subprocess.Popen(cmd,
418 shell=expand,
419 stdin=stdin_file,
420 stdout=subprocess.PIPE)
422 result = []
423 try:
424 while True:
425 entry = marshal.load(p4.stdout)
426 if cb is not None:
427 cb(entry)
428 else:
429 result.append(entry)
430 except EOFError:
431 pass
432 exitCode = p4.wait()
433 if exitCode != 0:
434 entry = {}
435 entry["p4ExitCode"] = exitCode
436 result.append(entry)
438 return result
440 def p4Cmd(cmd):
441 list = p4CmdList(cmd)
442 result = {}
443 for entry in list:
444 result.update(entry)
445 return result;
447 def p4Where(depotPath):
448 if not depotPath.endswith("/"):
449 depotPath += "/"
450 depotPath = depotPath + "..."
451 outputList = p4CmdList(["where", depotPath])
452 output = None
453 for entry in outputList:
454 if "depotFile" in entry:
455 if entry["depotFile"] == depotPath:
456 output = entry
457 break
458 elif "data" in entry:
459 data = entry.get("data")
460 space = data.find(" ")
461 if data[:space] == depotPath:
462 output = entry
463 break
464 if output == None:
465 return ""
466 if output["code"] == "error":
467 return ""
468 clientPath = ""
469 if "path" in output:
470 clientPath = output.get("path")
471 elif "data" in output:
472 data = output.get("data")
473 lastSpace = data.rfind(" ")
474 clientPath = data[lastSpace + 1:]
476 if clientPath.endswith("..."):
477 clientPath = clientPath[:-3]
478 return clientPath
480 def currentGitBranch():
481 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
483 def isValidGitDir(path):
484 if (os.path.exists(path + "/HEAD")
485 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
486 return True;
487 return False
489 def parseRevision(ref):
490 return read_pipe("git rev-parse %s" % ref).strip()
492 def branchExists(ref):
493 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
494 ignore_error=True)
495 return len(rev) > 0
497 def extractLogMessageFromGitCommit(commit):
498 logMessage = ""
500 ## fixme: title is first line of commit, not 1st paragraph.
501 foundTitle = False
502 for log in read_pipe_lines("git cat-file commit %s" % commit):
503 if not foundTitle:
504 if len(log) == 1:
505 foundTitle = True
506 continue
508 logMessage += log
509 return logMessage
511 def extractSettingsGitLog(log):
512 values = {}
513 for line in log.split("\n"):
514 line = line.strip()
515 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
516 if not m:
517 continue
519 assignments = m.group(1).split (':')
520 for a in assignments:
521 vals = a.split ('=')
522 key = vals[0].strip()
523 val = ('='.join (vals[1:])).strip()
524 if val.endswith ('\"') and val.startswith('"'):
525 val = val[1:-1]
527 values[key] = val
529 paths = values.get("depot-paths")
530 if not paths:
531 paths = values.get("depot-path")
532 if paths:
533 values['depot-paths'] = paths.split(',')
534 return values
536 def gitBranchExists(branch):
537 proc = subprocess.Popen(["git", "rev-parse", branch],
538 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
539 return proc.wait() == 0;
541 _gitConfig = {}
542 def gitConfig(key, args = None): # set args to "--bool", for instance
543 if not _gitConfig.has_key(key):
544 argsFilter = ""
545 if args != None:
546 argsFilter = "%s " % args
547 cmd = "git config %s%s" % (argsFilter, key)
548 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
549 return _gitConfig[key]
551 def gitConfigList(key):
552 if not _gitConfig.has_key(key):
553 _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
554 return _gitConfig[key]
556 def p4BranchesInGit(branchesAreInRemotes = True):
557 branches = {}
559 cmdline = "git rev-parse --symbolic "
560 if branchesAreInRemotes:
561 cmdline += " --remotes"
562 else:
563 cmdline += " --branches"
565 for line in read_pipe_lines(cmdline):
566 line = line.strip()
568 ## only import to p4/
569 if not line.startswith('p4/') or line == "p4/HEAD":
570 continue
571 branch = line
573 # strip off p4
574 branch = re.sub ("^p4/", "", line)
576 branches[branch] = parseRevision(line)
577 return branches
579 def findUpstreamBranchPoint(head = "HEAD"):
580 branches = p4BranchesInGit()
581 # map from depot-path to branch name
582 branchByDepotPath = {}
583 for branch in branches.keys():
584 tip = branches[branch]
585 log = extractLogMessageFromGitCommit(tip)
586 settings = extractSettingsGitLog(log)
587 if settings.has_key("depot-paths"):
588 paths = ",".join(settings["depot-paths"])
589 branchByDepotPath[paths] = "remotes/p4/" + branch
591 settings = None
592 parent = 0
593 while parent < 65535:
594 commit = head + "~%s" % parent
595 log = extractLogMessageFromGitCommit(commit)
596 settings = extractSettingsGitLog(log)
597 if settings.has_key("depot-paths"):
598 paths = ",".join(settings["depot-paths"])
599 if branchByDepotPath.has_key(paths):
600 return [branchByDepotPath[paths], settings]
602 parent = parent + 1
604 return ["", settings]
606 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
607 if not silent:
608 print ("Creating/updating branch(es) in %s based on origin branch(es)"
609 % localRefPrefix)
611 originPrefix = "origin/p4/"
613 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
614 line = line.strip()
615 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
616 continue
618 headName = line[len(originPrefix):]
619 remoteHead = localRefPrefix + headName
620 originHead = line
622 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
623 if (not original.has_key('depot-paths')
624 or not original.has_key('change')):
625 continue
627 update = False
628 if not gitBranchExists(remoteHead):
629 if verbose:
630 print "creating %s" % remoteHead
631 update = True
632 else:
633 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
634 if settings.has_key('change') > 0:
635 if settings['depot-paths'] == original['depot-paths']:
636 originP4Change = int(original['change'])
637 p4Change = int(settings['change'])
638 if originP4Change > p4Change:
639 print ("%s (%s) is newer than %s (%s). "
640 "Updating p4 branch from origin."
641 % (originHead, originP4Change,
642 remoteHead, p4Change))
643 update = True
644 else:
645 print ("Ignoring: %s was imported from %s while "
646 "%s was imported from %s"
647 % (originHead, ','.join(original['depot-paths']),
648 remoteHead, ','.join(settings['depot-paths'])))
650 if update:
651 system("git update-ref %s %s" % (remoteHead, originHead))
653 def originP4BranchesExist():
654 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
656 def p4ChangesForPaths(depotPaths, changeRange):
657 assert depotPaths
658 cmd = ['changes']
659 for p in depotPaths:
660 cmd += ["%s...%s" % (p, changeRange)]
661 output = p4_read_pipe_lines(cmd)
663 changes = {}
664 for line in output:
665 changeNum = int(line.split(" ")[1])
666 changes[changeNum] = True
668 changelist = changes.keys()
669 changelist.sort()
670 return changelist
672 def p4PathStartsWith(path, prefix):
673 # This method tries to remedy a potential mixed-case issue:
675 # If UserA adds //depot/DirA/file1
676 # and UserB adds //depot/dira/file2
678 # we may or may not have a problem. If you have core.ignorecase=true,
679 # we treat DirA and dira as the same directory
680 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
681 if ignorecase:
682 return path.lower().startswith(prefix.lower())
683 return path.startswith(prefix)
685 def getClientSpec():
686 """Look at the p4 client spec, create a View() object that contains
687 all the mappings, and return it."""
689 specList = p4CmdList("client -o")
690 if len(specList) != 1:
691 die('Output from "client -o" is %d lines, expecting 1' %
692 len(specList))
694 # dictionary of all client parameters
695 entry = specList[0]
697 # just the keys that start with "View"
698 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
700 # hold this new View
701 view = View()
703 # append the lines, in order, to the view
704 for view_num in range(len(view_keys)):
705 k = "View%d" % view_num
706 if k not in view_keys:
707 die("Expected view key %s missing" % k)
708 view.append(entry[k])
710 return view
712 def getClientRoot():
713 """Grab the client directory."""
715 output = p4CmdList("client -o")
716 if len(output) != 1:
717 die('Output from "client -o" is %d lines, expecting 1' % len(output))
719 entry = output[0]
720 if "Root" not in entry:
721 die('Client has no "Root"')
723 return entry["Root"]
726 # P4 wildcards are not allowed in filenames. P4 complains
727 # if you simply add them, but you can force it with "-f", in
728 # which case it translates them into %xx encoding internally.
730 def wildcard_decode(path):
731 # Search for and fix just these four characters. Do % last so
732 # that fixing it does not inadvertently create new %-escapes.
733 # Cannot have * in a filename in windows; untested as to
734 # what p4 would do in such a case.
735 if not platform.system() == "Windows":
736 path = path.replace("%2A", "*")
737 path = path.replace("%23", "#") \
738 .replace("%40", "@") \
739 .replace("%25", "%")
740 return path
742 def wildcard_encode(path):
743 # do % first to avoid double-encoding the %s introduced here
744 path = path.replace("%", "%25") \
745 .replace("*", "%2A") \
746 .replace("#", "%23") \
747 .replace("@", "%40")
748 return path
750 def wildcard_present(path):
751 return path.translate(None, "*#@%") != path
753 class Command:
754 def __init__(self):
755 self.usage = "usage: %prog [options]"
756 self.needsGit = True
757 self.verbose = False
759 class P4UserMap:
760 def __init__(self):
761 self.userMapFromPerforceServer = False
762 self.myP4UserId = None
764 def p4UserId(self):
765 if self.myP4UserId:
766 return self.myP4UserId
768 results = p4CmdList("user -o")
769 for r in results:
770 if r.has_key('User'):
771 self.myP4UserId = r['User']
772 return r['User']
773 die("Could not find your p4 user id")
775 def p4UserIsMe(self, p4User):
776 # return True if the given p4 user is actually me
777 me = self.p4UserId()
778 if not p4User or p4User != me:
779 return False
780 else:
781 return True
783 def getUserCacheFilename(self):
784 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
785 return home + "/.gitp4-usercache.txt"
787 def getUserMapFromPerforceServer(self):
788 if self.userMapFromPerforceServer:
789 return
790 self.users = {}
791 self.emails = {}
793 for output in p4CmdList("users"):
794 if not output.has_key("User"):
795 continue
796 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
797 self.emails[output["Email"]] = output["User"]
800 s = ''
801 for (key, val) in self.users.items():
802 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
804 open(self.getUserCacheFilename(), "wb").write(s)
805 self.userMapFromPerforceServer = True
807 def loadUserMapFromCache(self):
808 self.users = {}
809 self.userMapFromPerforceServer = False
810 try:
811 cache = open(self.getUserCacheFilename(), "rb")
812 lines = cache.readlines()
813 cache.close()
814 for line in lines:
815 entry = line.strip().split("\t")
816 self.users[entry[0]] = entry[1]
817 except IOError:
818 self.getUserMapFromPerforceServer()
820 class P4Debug(Command):
821 def __init__(self):
822 Command.__init__(self)
823 self.options = []
824 self.description = "A tool to debug the output of p4 -G."
825 self.needsGit = False
827 def run(self, args):
828 j = 0
829 for output in p4CmdList(args):
830 print 'Element: %d' % j
831 j += 1
832 print output
833 return True
835 class P4RollBack(Command):
836 def __init__(self):
837 Command.__init__(self)
838 self.options = [
839 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
841 self.description = "A tool to debug the multi-branch import. Don't use :)"
842 self.rollbackLocalBranches = False
844 def run(self, args):
845 if len(args) != 1:
846 return False
847 maxChange = int(args[0])
849 if "p4ExitCode" in p4Cmd("changes -m 1"):
850 die("Problems executing p4");
852 if self.rollbackLocalBranches:
853 refPrefix = "refs/heads/"
854 lines = read_pipe_lines("git rev-parse --symbolic --branches")
855 else:
856 refPrefix = "refs/remotes/"
857 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
859 for line in lines:
860 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
861 line = line.strip()
862 ref = refPrefix + line
863 log = extractLogMessageFromGitCommit(ref)
864 settings = extractSettingsGitLog(log)
866 depotPaths = settings['depot-paths']
867 change = settings['change']
869 changed = False
871 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
872 for p in depotPaths]))) == 0:
873 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
874 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
875 continue
877 while change and int(change) > maxChange:
878 changed = True
879 if self.verbose:
880 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
881 system("git update-ref %s \"%s^\"" % (ref, ref))
882 log = extractLogMessageFromGitCommit(ref)
883 settings = extractSettingsGitLog(log)
886 depotPaths = settings['depot-paths']
887 change = settings['change']
889 if changed:
890 print "%s rewound to %s" % (ref, change)
892 return True
894 class P4Submit(Command, P4UserMap):
896 conflict_behavior_choices = ("ask", "skip", "quit")
898 def __init__(self):
899 Command.__init__(self)
900 P4UserMap.__init__(self)
901 self.options = [
902 optparse.make_option("--origin", dest="origin"),
903 optparse.make_option("-M", dest="detectRenames", action="store_true"),
904 # preserve the user, requires relevant p4 permissions
905 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
906 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
907 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
908 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
909 optparse.make_option("--conflict", dest="conflict_behavior",
910 choices=self.conflict_behavior_choices)
912 self.description = "Submit changes from git to the perforce depot."
913 self.usage += " [name of git branch to submit into perforce depot]"
914 self.origin = ""
915 self.detectRenames = False
916 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
917 self.dry_run = False
918 self.prepare_p4_only = False
919 self.conflict_behavior = None
920 self.isWindows = (platform.system() == "Windows")
921 self.exportLabels = False
922 self.p4HasMoveCommand = p4_has_move_command()
924 def check(self):
925 if len(p4CmdList("opened ...")) > 0:
926 die("You have files opened with perforce! Close them before starting the sync.")
928 def separate_jobs_from_description(self, message):
929 """Extract and return a possible Jobs field in the commit
930 message. It goes into a separate section in the p4 change
931 specification.
933 A jobs line starts with "Jobs:" and looks like a new field
934 in a form. Values are white-space separated on the same
935 line or on following lines that start with a tab.
937 This does not parse and extract the full git commit message
938 like a p4 form. It just sees the Jobs: line as a marker
939 to pass everything from then on directly into the p4 form,
940 but outside the description section.
942 Return a tuple (stripped log message, jobs string)."""
944 m = re.search(r'^Jobs:', message, re.MULTILINE)
945 if m is None:
946 return (message, None)
948 jobtext = message[m.start():]
949 stripped_message = message[:m.start()].rstrip()
950 return (stripped_message, jobtext)
952 def prepareLogMessage(self, template, message, jobs):
953 """Edits the template returned from "p4 change -o" to insert
954 the message in the Description field, and the jobs text in
955 the Jobs field."""
956 result = ""
958 inDescriptionSection = False
960 for line in template.split("\n"):
961 if line.startswith("#"):
962 result += line + "\n"
963 continue
965 if inDescriptionSection:
966 if line.startswith("Files:") or line.startswith("Jobs:"):
967 inDescriptionSection = False
968 # insert Jobs section
969 if jobs:
970 result += jobs + "\n"
971 else:
972 continue
973 else:
974 if line.startswith("Description:"):
975 inDescriptionSection = True
976 line += "\n"
977 for messageLine in message.split("\n"):
978 line += "\t" + messageLine + "\n"
980 result += line + "\n"
982 return result
984 def patchRCSKeywords(self, file, pattern):
985 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
986 (handle, outFileName) = tempfile.mkstemp(dir='.')
987 try:
988 outFile = os.fdopen(handle, "w+")
989 inFile = open(file, "r")
990 regexp = re.compile(pattern, re.VERBOSE)
991 for line in inFile.readlines():
992 line = regexp.sub(r'$\1$', line)
993 outFile.write(line)
994 inFile.close()
995 outFile.close()
996 # Forcibly overwrite the original file
997 os.unlink(file)
998 shutil.move(outFileName, file)
999 except:
1000 # cleanup our temporary file
1001 os.unlink(outFileName)
1002 print "Failed to strip RCS keywords in %s" % file
1003 raise
1005 print "Patched up RCS keywords in %s" % file
1007 def p4UserForCommit(self,id):
1008 # Return the tuple (perforce user,git email) for a given git commit id
1009 self.getUserMapFromPerforceServer()
1010 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
1011 gitEmail = gitEmail.strip()
1012 if not self.emails.has_key(gitEmail):
1013 return (None,gitEmail)
1014 else:
1015 return (self.emails[gitEmail],gitEmail)
1017 def checkValidP4Users(self,commits):
1018 # check if any git authors cannot be mapped to p4 users
1019 for id in commits:
1020 (user,email) = self.p4UserForCommit(id)
1021 if not user:
1022 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1023 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
1024 print "%s" % msg
1025 else:
1026 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1028 def lastP4Changelist(self):
1029 # Get back the last changelist number submitted in this client spec. This
1030 # then gets used to patch up the username in the change. If the same
1031 # client spec is being used by multiple processes then this might go
1032 # wrong.
1033 results = p4CmdList("client -o") # find the current client
1034 client = None
1035 for r in results:
1036 if r.has_key('Client'):
1037 client = r['Client']
1038 break
1039 if not client:
1040 die("could not get client spec")
1041 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1042 for r in results:
1043 if r.has_key('change'):
1044 return r['change']
1045 die("Could not get changelist number for last submit - cannot patch up user details")
1047 def modifyChangelistUser(self, changelist, newUser):
1048 # fixup the user field of a changelist after it has been submitted.
1049 changes = p4CmdList("change -o %s" % changelist)
1050 if len(changes) != 1:
1051 die("Bad output from p4 change modifying %s to user %s" %
1052 (changelist, newUser))
1054 c = changes[0]
1055 if c['User'] == newUser: return # nothing to do
1056 c['User'] = newUser
1057 input = marshal.dumps(c)
1059 result = p4CmdList("change -f -i", stdin=input)
1060 for r in result:
1061 if r.has_key('code'):
1062 if r['code'] == 'error':
1063 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1064 if r.has_key('data'):
1065 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1066 return
1067 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1069 def canChangeChangelists(self):
1070 # check to see if we have p4 admin or super-user permissions, either of
1071 # which are required to modify changelists.
1072 results = p4CmdList(["protects", self.depotPath])
1073 for r in results:
1074 if r.has_key('perm'):
1075 if r['perm'] == 'admin':
1076 return 1
1077 if r['perm'] == 'super':
1078 return 1
1079 return 0
1081 def prepareSubmitTemplate(self):
1082 """Run "p4 change -o" to grab a change specification template.
1083 This does not use "p4 -G", as it is nice to keep the submission
1084 template in original order, since a human might edit it.
1086 Remove lines in the Files section that show changes to files
1087 outside the depot path we're committing into."""
1089 template = ""
1090 inFilesSection = False
1091 for line in p4_read_pipe_lines(['change', '-o']):
1092 if line.endswith("\r\n"):
1093 line = line[:-2] + "\n"
1094 if inFilesSection:
1095 if line.startswith("\t"):
1096 # path starts and ends with a tab
1097 path = line[1:]
1098 lastTab = path.rfind("\t")
1099 if lastTab != -1:
1100 path = path[:lastTab]
1101 if not p4PathStartsWith(path, self.depotPath):
1102 continue
1103 else:
1104 inFilesSection = False
1105 else:
1106 if line.startswith("Files:"):
1107 inFilesSection = True
1109 template += line
1111 return template
1113 def edit_template(self, template_file):
1114 """Invoke the editor to let the user change the submission
1115 message. Return true if okay to continue with the submit."""
1117 # if configured to skip the editing part, just submit
1118 if gitConfig("git-p4.skipSubmitEdit") == "true":
1119 return True
1121 # look at the modification time, to check later if the user saved
1122 # the file
1123 mtime = os.stat(template_file).st_mtime
1125 # invoke the editor
1126 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1127 editor = os.environ.get("P4EDITOR")
1128 else:
1129 editor = read_pipe("git var GIT_EDITOR").strip()
1130 system(editor + " " + template_file)
1132 # If the file was not saved, prompt to see if this patch should
1133 # be skipped. But skip this verification step if configured so.
1134 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1135 return True
1137 # modification time updated means user saved the file
1138 if os.stat(template_file).st_mtime > mtime:
1139 return True
1141 while True:
1142 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1143 if response == 'y':
1144 return True
1145 if response == 'n':
1146 return False
1148 def applyCommit(self, id):
1149 """Apply one commit, return True if it succeeded."""
1151 print "Applying", read_pipe(["git", "show", "-s",
1152 "--format=format:%h %s", id])
1154 (p4User, gitEmail) = self.p4UserForCommit(id)
1156 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1157 filesToAdd = set()
1158 filesToDelete = set()
1159 editedFiles = set()
1160 pureRenameCopy = set()
1161 filesToChangeExecBit = {}
1163 for line in diff:
1164 diff = parseDiffTreeEntry(line)
1165 modifier = diff['status']
1166 path = diff['src']
1167 if modifier == "M":
1168 p4_edit(path)
1169 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1170 filesToChangeExecBit[path] = diff['dst_mode']
1171 editedFiles.add(path)
1172 elif modifier == "A":
1173 filesToAdd.add(path)
1174 filesToChangeExecBit[path] = diff['dst_mode']
1175 if path in filesToDelete:
1176 filesToDelete.remove(path)
1177 elif modifier == "D":
1178 filesToDelete.add(path)
1179 if path in filesToAdd:
1180 filesToAdd.remove(path)
1181 elif modifier == "C":
1182 src, dest = diff['src'], diff['dst']
1183 p4_integrate(src, dest)
1184 pureRenameCopy.add(dest)
1185 if diff['src_sha1'] != diff['dst_sha1']:
1186 p4_edit(dest)
1187 pureRenameCopy.discard(dest)
1188 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1189 p4_edit(dest)
1190 pureRenameCopy.discard(dest)
1191 filesToChangeExecBit[dest] = diff['dst_mode']
1192 os.unlink(dest)
1193 editedFiles.add(dest)
1194 elif modifier == "R":
1195 src, dest = diff['src'], diff['dst']
1196 if self.p4HasMoveCommand:
1197 p4_edit(src) # src must be open before move
1198 p4_move(src, dest) # opens for (move/delete, move/add)
1199 else:
1200 p4_integrate(src, dest)
1201 if diff['src_sha1'] != diff['dst_sha1']:
1202 p4_edit(dest)
1203 else:
1204 pureRenameCopy.add(dest)
1205 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1206 if not self.p4HasMoveCommand:
1207 p4_edit(dest) # with move: already open, writable
1208 filesToChangeExecBit[dest] = diff['dst_mode']
1209 if not self.p4HasMoveCommand:
1210 os.unlink(dest)
1211 filesToDelete.add(src)
1212 editedFiles.add(dest)
1213 else:
1214 die("unknown modifier %s for %s" % (modifier, path))
1216 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1217 patchcmd = diffcmd + " | git apply "
1218 tryPatchCmd = patchcmd + "--check -"
1219 applyPatchCmd = patchcmd + "--check --apply -"
1220 patch_succeeded = True
1222 if os.system(tryPatchCmd) != 0:
1223 fixed_rcs_keywords = False
1224 patch_succeeded = False
1225 print "Unfortunately applying the change failed!"
1227 # Patch failed, maybe it's just RCS keyword woes. Look through
1228 # the patch to see if that's possible.
1229 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1230 file = None
1231 pattern = None
1232 kwfiles = {}
1233 for file in editedFiles | filesToDelete:
1234 # did this file's delta contain RCS keywords?
1235 pattern = p4_keywords_regexp_for_file(file)
1237 if pattern:
1238 # this file is a possibility...look for RCS keywords.
1239 regexp = re.compile(pattern, re.VERBOSE)
1240 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1241 if regexp.search(line):
1242 if verbose:
1243 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1244 kwfiles[file] = pattern
1245 break
1247 for file in kwfiles:
1248 if verbose:
1249 print "zapping %s with %s" % (line,pattern)
1250 self.patchRCSKeywords(file, kwfiles[file])
1251 fixed_rcs_keywords = True
1253 if fixed_rcs_keywords:
1254 print "Retrying the patch with RCS keywords cleaned up"
1255 if os.system(tryPatchCmd) == 0:
1256 patch_succeeded = True
1258 if not patch_succeeded:
1259 for f in editedFiles:
1260 p4_revert(f)
1261 return False
1264 # Apply the patch for real, and do add/delete/+x handling.
1266 system(applyPatchCmd)
1268 for f in filesToAdd:
1269 p4_add(f)
1270 for f in filesToDelete:
1271 p4_revert(f)
1272 p4_delete(f)
1274 # Set/clear executable bits
1275 for f in filesToChangeExecBit.keys():
1276 mode = filesToChangeExecBit[f]
1277 setP4ExecBit(f, mode)
1280 # Build p4 change description, starting with the contents
1281 # of the git commit message.
1283 logMessage = extractLogMessageFromGitCommit(id)
1284 logMessage = logMessage.strip()
1285 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1287 template = self.prepareSubmitTemplate()
1288 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1290 if self.preserveUser:
1291 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1293 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1294 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1295 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1296 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1298 separatorLine = "######## everything below this line is just the diff #######\n"
1300 # diff
1301 if os.environ.has_key("P4DIFF"):
1302 del(os.environ["P4DIFF"])
1303 diff = ""
1304 for editedFile in editedFiles:
1305 diff += p4_read_pipe(['diff', '-du',
1306 wildcard_encode(editedFile)])
1308 # new file diff
1309 newdiff = ""
1310 for newFile in filesToAdd:
1311 newdiff += "==== new file ====\n"
1312 newdiff += "--- /dev/null\n"
1313 newdiff += "+++ %s\n" % newFile
1314 f = open(newFile, "r")
1315 for line in f.readlines():
1316 newdiff += "+" + line
1317 f.close()
1319 # change description file: submitTemplate, separatorLine, diff, newdiff
1320 (handle, fileName) = tempfile.mkstemp()
1321 tmpFile = os.fdopen(handle, "w+")
1322 if self.isWindows:
1323 submitTemplate = submitTemplate.replace("\n", "\r\n")
1324 separatorLine = separatorLine.replace("\n", "\r\n")
1325 newdiff = newdiff.replace("\n", "\r\n")
1326 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1327 tmpFile.close()
1329 if self.prepare_p4_only:
1331 # Leave the p4 tree prepared, and the submit template around
1332 # and let the user decide what to do next
1334 print
1335 print "P4 workspace prepared for submission."
1336 print "To submit or revert, go to client workspace"
1337 print " " + self.clientPath
1338 print
1339 print "To submit, use \"p4 submit\" to write a new description,"
1340 print "or \"p4 submit -i %s\" to use the one prepared by" \
1341 " \"git p4\"." % fileName
1342 print "You can delete the file \"%s\" when finished." % fileName
1344 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1345 print "To preserve change ownership by user %s, you must\n" \
1346 "do \"p4 change -f <change>\" after submitting and\n" \
1347 "edit the User field."
1348 if pureRenameCopy:
1349 print "After submitting, renamed files must be re-synced."
1350 print "Invoke \"p4 sync -f\" on each of these files:"
1351 for f in pureRenameCopy:
1352 print " " + f
1354 print
1355 print "To revert the changes, use \"p4 revert ...\", and delete"
1356 print "the submit template file \"%s\"" % fileName
1357 if filesToAdd:
1358 print "Since the commit adds new files, they must be deleted:"
1359 for f in filesToAdd:
1360 print " " + f
1361 print
1362 return True
1365 # Let the user edit the change description, then submit it.
1367 if self.edit_template(fileName):
1368 # read the edited message and submit
1369 ret = True
1370 tmpFile = open(fileName, "rb")
1371 message = tmpFile.read()
1372 tmpFile.close()
1373 submitTemplate = message[:message.index(separatorLine)]
1374 if self.isWindows:
1375 submitTemplate = submitTemplate.replace("\r\n", "\n")
1376 p4_write_pipe(['submit', '-i'], submitTemplate)
1378 if self.preserveUser:
1379 if p4User:
1380 # Get last changelist number. Cannot easily get it from
1381 # the submit command output as the output is
1382 # unmarshalled.
1383 changelist = self.lastP4Changelist()
1384 self.modifyChangelistUser(changelist, p4User)
1386 # The rename/copy happened by applying a patch that created a
1387 # new file. This leaves it writable, which confuses p4.
1388 for f in pureRenameCopy:
1389 p4_sync(f, "-f")
1391 else:
1392 # skip this patch
1393 ret = False
1394 print "Submission cancelled, undoing p4 changes."
1395 for f in editedFiles:
1396 p4_revert(f)
1397 for f in filesToAdd:
1398 p4_revert(f)
1399 os.remove(f)
1400 for f in filesToDelete:
1401 p4_revert(f)
1403 os.remove(fileName)
1404 return ret
1406 # Export git tags as p4 labels. Create a p4 label and then tag
1407 # with that.
1408 def exportGitTags(self, gitTags):
1409 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1410 if len(validLabelRegexp) == 0:
1411 validLabelRegexp = defaultLabelRegexp
1412 m = re.compile(validLabelRegexp)
1414 for name in gitTags:
1416 if not m.match(name):
1417 if verbose:
1418 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1419 continue
1421 # Get the p4 commit this corresponds to
1422 logMessage = extractLogMessageFromGitCommit(name)
1423 values = extractSettingsGitLog(logMessage)
1425 if not values.has_key('change'):
1426 # a tag pointing to something not sent to p4; ignore
1427 if verbose:
1428 print "git tag %s does not give a p4 commit" % name
1429 continue
1430 else:
1431 changelist = values['change']
1433 # Get the tag details.
1434 inHeader = True
1435 isAnnotated = False
1436 body = []
1437 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1438 l = l.strip()
1439 if inHeader:
1440 if re.match(r'tag\s+', l):
1441 isAnnotated = True
1442 elif re.match(r'\s*$', l):
1443 inHeader = False
1444 continue
1445 else:
1446 body.append(l)
1448 if not isAnnotated:
1449 body = ["lightweight tag imported by git p4\n"]
1451 # Create the label - use the same view as the client spec we are using
1452 clientSpec = getClientSpec()
1454 labelTemplate = "Label: %s\n" % name
1455 labelTemplate += "Description:\n"
1456 for b in body:
1457 labelTemplate += "\t" + b + "\n"
1458 labelTemplate += "View:\n"
1459 for mapping in clientSpec.mappings:
1460 labelTemplate += "\t%s\n" % mapping.depot_side.path
1462 if self.dry_run:
1463 print "Would create p4 label %s for tag" % name
1464 elif self.prepare_p4_only:
1465 print "Not creating p4 label %s for tag due to option" \
1466 " --prepare-p4-only" % name
1467 else:
1468 p4_write_pipe(["label", "-i"], labelTemplate)
1470 # Use the label
1471 p4_system(["tag", "-l", name] +
1472 ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1474 if verbose:
1475 print "created p4 label for tag %s" % name
1477 def run(self, args):
1478 if len(args) == 0:
1479 self.master = currentGitBranch()
1480 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1481 die("Detecting current git branch failed!")
1482 elif len(args) == 1:
1483 self.master = args[0]
1484 if not branchExists(self.master):
1485 die("Branch %s does not exist" % self.master)
1486 else:
1487 return False
1489 allowSubmit = gitConfig("git-p4.allowSubmit")
1490 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1491 die("%s is not in git-p4.allowSubmit" % self.master)
1493 [upstream, settings] = findUpstreamBranchPoint()
1494 self.depotPath = settings['depot-paths'][0]
1495 if len(self.origin) == 0:
1496 self.origin = upstream
1498 if self.preserveUser:
1499 if not self.canChangeChangelists():
1500 die("Cannot preserve user names without p4 super-user or admin permissions")
1502 # if not set from the command line, try the config file
1503 if self.conflict_behavior is None:
1504 val = gitConfig("git-p4.conflict")
1505 if val:
1506 if val not in self.conflict_behavior_choices:
1507 die("Invalid value '%s' for config git-p4.conflict" % val)
1508 else:
1509 val = "ask"
1510 self.conflict_behavior = val
1512 if self.verbose:
1513 print "Origin branch is " + self.origin
1515 if len(self.depotPath) == 0:
1516 print "Internal error: cannot locate perforce depot path from existing branches"
1517 sys.exit(128)
1519 self.useClientSpec = False
1520 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1521 self.useClientSpec = True
1522 if self.useClientSpec:
1523 self.clientSpecDirs = getClientSpec()
1525 if self.useClientSpec:
1526 # all files are relative to the client spec
1527 self.clientPath = getClientRoot()
1528 else:
1529 self.clientPath = p4Where(self.depotPath)
1531 if self.clientPath == "":
1532 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1534 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1535 self.oldWorkingDirectory = os.getcwd()
1537 # ensure the clientPath exists
1538 new_client_dir = False
1539 if not os.path.exists(self.clientPath):
1540 new_client_dir = True
1541 os.makedirs(self.clientPath)
1543 chdir(self.clientPath)
1544 if self.dry_run:
1545 print "Would synchronize p4 checkout in %s" % self.clientPath
1546 else:
1547 print "Synchronizing p4 checkout..."
1548 if new_client_dir:
1549 # old one was destroyed, and maybe nobody told p4
1550 p4_sync("...", "-f")
1551 else:
1552 p4_sync("...")
1553 self.check()
1555 commits = []
1556 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1557 commits.append(line.strip())
1558 commits.reverse()
1560 if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1561 self.checkAuthorship = False
1562 else:
1563 self.checkAuthorship = True
1565 if self.preserveUser:
1566 self.checkValidP4Users(commits)
1569 # Build up a set of options to be passed to diff when
1570 # submitting each commit to p4.
1572 if self.detectRenames:
1573 # command-line -M arg
1574 self.diffOpts = "-M"
1575 else:
1576 # If not explicitly set check the config variable
1577 detectRenames = gitConfig("git-p4.detectRenames")
1579 if detectRenames.lower() == "false" or detectRenames == "":
1580 self.diffOpts = ""
1581 elif detectRenames.lower() == "true":
1582 self.diffOpts = "-M"
1583 else:
1584 self.diffOpts = "-M%s" % detectRenames
1586 # no command-line arg for -C or --find-copies-harder, just
1587 # config variables
1588 detectCopies = gitConfig("git-p4.detectCopies")
1589 if detectCopies.lower() == "false" or detectCopies == "":
1590 pass
1591 elif detectCopies.lower() == "true":
1592 self.diffOpts += " -C"
1593 else:
1594 self.diffOpts += " -C%s" % detectCopies
1596 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1597 self.diffOpts += " --find-copies-harder"
1600 # Apply the commits, one at a time. On failure, ask if should
1601 # continue to try the rest of the patches, or quit.
1603 if self.dry_run:
1604 print "Would apply"
1605 applied = []
1606 last = len(commits) - 1
1607 for i, commit in enumerate(commits):
1608 if self.dry_run:
1609 print " ", read_pipe(["git", "show", "-s",
1610 "--format=format:%h %s", commit])
1611 ok = True
1612 else:
1613 ok = self.applyCommit(commit)
1614 if ok:
1615 applied.append(commit)
1616 else:
1617 if self.prepare_p4_only and i < last:
1618 print "Processing only the first commit due to option" \
1619 " --prepare-p4-only"
1620 break
1621 if i < last:
1622 quit = False
1623 while True:
1624 # prompt for what to do, or use the option/variable
1625 if self.conflict_behavior == "ask":
1626 print "What do you want to do?"
1627 response = raw_input("[s]kip this commit but apply"
1628 " the rest, or [q]uit? ")
1629 if not response:
1630 continue
1631 elif self.conflict_behavior == "skip":
1632 response = "s"
1633 elif self.conflict_behavior == "quit":
1634 response = "q"
1635 else:
1636 die("Unknown conflict_behavior '%s'" %
1637 self.conflict_behavior)
1639 if response[0] == "s":
1640 print "Skipping this commit, but applying the rest"
1641 break
1642 if response[0] == "q":
1643 print "Quitting"
1644 quit = True
1645 break
1646 if quit:
1647 break
1649 chdir(self.oldWorkingDirectory)
1651 if self.dry_run:
1652 pass
1653 elif self.prepare_p4_only:
1654 pass
1655 elif len(commits) == len(applied):
1656 print "All commits applied!"
1658 sync = P4Sync()
1659 sync.run([])
1661 rebase = P4Rebase()
1662 rebase.rebase()
1664 else:
1665 if len(applied) == 0:
1666 print "No commits applied."
1667 else:
1668 print "Applied only the commits marked with '*':"
1669 for c in commits:
1670 if c in applied:
1671 star = "*"
1672 else:
1673 star = " "
1674 print star, read_pipe(["git", "show", "-s",
1675 "--format=format:%h %s", c])
1676 print "You will have to do 'git p4 sync' and rebase."
1678 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1679 self.exportLabels = True
1681 if self.exportLabels:
1682 p4Labels = getP4Labels(self.depotPath)
1683 gitTags = getGitTags()
1685 missingGitTags = gitTags - p4Labels
1686 self.exportGitTags(missingGitTags)
1688 # exit with error unless everything applied perfecly
1689 if len(commits) != len(applied):
1690 sys.exit(1)
1692 return True
1694 class View(object):
1695 """Represent a p4 view ("p4 help views"), and map files in a
1696 repo according to the view."""
1698 class Path(object):
1699 """A depot or client path, possibly containing wildcards.
1700 The only one supported is ... at the end, currently.
1701 Initialize with the full path, with //depot or //client."""
1703 def __init__(self, path, is_depot):
1704 self.path = path
1705 self.is_depot = is_depot
1706 self.find_wildcards()
1707 # remember the prefix bit, useful for relative mappings
1708 m = re.match("(//[^/]+/)", self.path)
1709 if not m:
1710 die("Path %s does not start with //prefix/" % self.path)
1711 prefix = m.group(1)
1712 if not self.is_depot:
1713 # strip //client/ on client paths
1714 self.path = self.path[len(prefix):]
1716 def find_wildcards(self):
1717 """Make sure wildcards are valid, and set up internal
1718 variables."""
1720 self.ends_triple_dot = False
1721 # There are three wildcards allowed in p4 views
1722 # (see "p4 help views"). This code knows how to
1723 # handle "..." (only at the end), but cannot deal with
1724 # "%%n" or "*". Only check the depot_side, as p4 should
1725 # validate that the client_side matches too.
1726 if re.search(r'%%[1-9]', self.path):
1727 die("Can't handle %%n wildcards in view: %s" % self.path)
1728 if self.path.find("*") >= 0:
1729 die("Can't handle * wildcards in view: %s" % self.path)
1730 triple_dot_index = self.path.find("...")
1731 if triple_dot_index >= 0:
1732 if triple_dot_index != len(self.path) - 3:
1733 die("Can handle only single ... wildcard, at end: %s" %
1734 self.path)
1735 self.ends_triple_dot = True
1737 def ensure_compatible(self, other_path):
1738 """Make sure the wildcards agree."""
1739 if self.ends_triple_dot != other_path.ends_triple_dot:
1740 die("Both paths must end with ... if either does;\n" +
1741 "paths: %s %s" % (self.path, other_path.path))
1743 def match_wildcards(self, test_path):
1744 """See if this test_path matches us, and fill in the value
1745 of the wildcards if so. Returns a tuple of
1746 (True|False, wildcards[]). For now, only the ... at end
1747 is supported, so at most one wildcard."""
1748 if self.ends_triple_dot:
1749 dotless = self.path[:-3]
1750 if test_path.startswith(dotless):
1751 wildcard = test_path[len(dotless):]
1752 return (True, [ wildcard ])
1753 else:
1754 if test_path == self.path:
1755 return (True, [])
1756 return (False, [])
1758 def match(self, test_path):
1759 """Just return if it matches; don't bother with the wildcards."""
1760 b, _ = self.match_wildcards(test_path)
1761 return b
1763 def fill_in_wildcards(self, wildcards):
1764 """Return the relative path, with the wildcards filled in
1765 if there are any."""
1766 if self.ends_triple_dot:
1767 return self.path[:-3] + wildcards[0]
1768 else:
1769 return self.path
1771 class Mapping(object):
1772 def __init__(self, depot_side, client_side, overlay, exclude):
1773 # depot_side is without the trailing /... if it had one
1774 self.depot_side = View.Path(depot_side, is_depot=True)
1775 self.client_side = View.Path(client_side, is_depot=False)
1776 self.overlay = overlay # started with "+"
1777 self.exclude = exclude # started with "-"
1778 assert not (self.overlay and self.exclude)
1779 self.depot_side.ensure_compatible(self.client_side)
1781 def __str__(self):
1782 c = " "
1783 if self.overlay:
1784 c = "+"
1785 if self.exclude:
1786 c = "-"
1787 return "View.Mapping: %s%s -> %s" % \
1788 (c, self.depot_side.path, self.client_side.path)
1790 def map_depot_to_client(self, depot_path):
1791 """Calculate the client path if using this mapping on the
1792 given depot path; does not consider the effect of other
1793 mappings in a view. Even excluded mappings are returned."""
1794 matches, wildcards = self.depot_side.match_wildcards(depot_path)
1795 if not matches:
1796 return ""
1797 client_path = self.client_side.fill_in_wildcards(wildcards)
1798 return client_path
1801 # View methods
1803 def __init__(self):
1804 self.mappings = []
1806 def append(self, view_line):
1807 """Parse a view line, splitting it into depot and client
1808 sides. Append to self.mappings, preserving order."""
1810 # Split the view line into exactly two words. P4 enforces
1811 # structure on these lines that simplifies this quite a bit.
1813 # Either or both words may be double-quoted.
1814 # Single quotes do not matter.
1815 # Double-quote marks cannot occur inside the words.
1816 # A + or - prefix is also inside the quotes.
1817 # There are no quotes unless they contain a space.
1818 # The line is already white-space stripped.
1819 # The two words are separated by a single space.
1821 if view_line[0] == '"':
1822 # First word is double quoted. Find its end.
1823 close_quote_index = view_line.find('"', 1)
1824 if close_quote_index <= 0:
1825 die("No first-word closing quote found: %s" % view_line)
1826 depot_side = view_line[1:close_quote_index]
1827 # skip closing quote and space
1828 rhs_index = close_quote_index + 1 + 1
1829 else:
1830 space_index = view_line.find(" ")
1831 if space_index <= 0:
1832 die("No word-splitting space found: %s" % view_line)
1833 depot_side = view_line[0:space_index]
1834 rhs_index = space_index + 1
1836 if view_line[rhs_index] == '"':
1837 # Second word is double quoted. Make sure there is a
1838 # double quote at the end too.
1839 if not view_line.endswith('"'):
1840 die("View line with rhs quote should end with one: %s" %
1841 view_line)
1842 # skip the quotes
1843 client_side = view_line[rhs_index+1:-1]
1844 else:
1845 client_side = view_line[rhs_index:]
1847 # prefix + means overlay on previous mapping
1848 overlay = False
1849 if depot_side.startswith("+"):
1850 overlay = True
1851 depot_side = depot_side[1:]
1853 # prefix - means exclude this path
1854 exclude = False
1855 if depot_side.startswith("-"):
1856 exclude = True
1857 depot_side = depot_side[1:]
1859 m = View.Mapping(depot_side, client_side, overlay, exclude)
1860 self.mappings.append(m)
1862 def map_in_client(self, depot_path):
1863 """Return the relative location in the client where this
1864 depot file should live. Returns "" if the file should
1865 not be mapped in the client."""
1867 paths_filled = []
1868 client_path = ""
1870 # look at later entries first
1871 for m in self.mappings[::-1]:
1873 # see where will this path end up in the client
1874 p = m.map_depot_to_client(depot_path)
1876 if p == "":
1877 # Depot path does not belong in client. Must remember
1878 # this, as previous items should not cause files to
1879 # exist in this path either. Remember that the list is
1880 # being walked from the end, which has higher precedence.
1881 # Overlap mappings do not exclude previous mappings.
1882 if not m.overlay:
1883 paths_filled.append(m.client_side)
1885 else:
1886 # This mapping matched; no need to search any further.
1887 # But, the mapping could be rejected if the client path
1888 # has already been claimed by an earlier mapping (i.e.
1889 # one later in the list, which we are walking backwards).
1890 already_mapped_in_client = False
1891 for f in paths_filled:
1892 # this is View.Path.match
1893 if f.match(p):
1894 already_mapped_in_client = True
1895 break
1896 if not already_mapped_in_client:
1897 # Include this file, unless it is from a line that
1898 # explicitly said to exclude it.
1899 if not m.exclude:
1900 client_path = p
1902 # a match, even if rejected, always stops the search
1903 break
1905 return client_path
1907 class P4Sync(Command, P4UserMap):
1908 delete_actions = ( "delete", "move/delete", "purge" )
1910 def __init__(self):
1911 Command.__init__(self)
1912 P4UserMap.__init__(self)
1913 self.options = [
1914 optparse.make_option("--branch", dest="branch"),
1915 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1916 optparse.make_option("--changesfile", dest="changesFile"),
1917 optparse.make_option("--silent", dest="silent", action="store_true"),
1918 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1919 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1920 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1921 help="Import into refs/heads/ , not refs/remotes"),
1922 optparse.make_option("--max-changes", dest="maxChanges"),
1923 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1924 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1925 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1926 help="Only sync files that are included in the Perforce Client Spec")
1928 self.description = """Imports from Perforce into a git repository.\n
1929 example:
1930 //depot/my/project/ -- to import the current head
1931 //depot/my/project/@all -- to import everything
1932 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1934 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1936 self.usage += " //depot/path[@revRange]"
1937 self.silent = False
1938 self.createdBranches = set()
1939 self.committedChanges = set()
1940 self.branch = ""
1941 self.detectBranches = False
1942 self.detectLabels = False
1943 self.importLabels = False
1944 self.changesFile = ""
1945 self.syncWithOrigin = True
1946 self.importIntoRemotes = True
1947 self.maxChanges = ""
1948 self.isWindows = (platform.system() == "Windows")
1949 self.keepRepoPath = False
1950 self.depotPaths = None
1951 self.p4BranchesInGit = []
1952 self.cloneExclude = []
1953 self.useClientSpec = False
1954 self.useClientSpec_from_options = False
1955 self.clientSpecDirs = None
1956 self.tempBranches = []
1957 self.tempBranchLocation = "git-p4-tmp"
1959 if gitConfig("git-p4.syncFromOrigin") == "false":
1960 self.syncWithOrigin = False
1962 # Force a checkpoint in fast-import and wait for it to finish
1963 def checkpoint(self):
1964 self.gitStream.write("checkpoint\n\n")
1965 self.gitStream.write("progress checkpoint\n\n")
1966 out = self.gitOutput.readline()
1967 if self.verbose:
1968 print "checkpoint finished: " + out
1970 def extractFilesFromCommit(self, commit):
1971 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1972 for path in self.cloneExclude]
1973 files = []
1974 fnum = 0
1975 while commit.has_key("depotFile%s" % fnum):
1976 path = commit["depotFile%s" % fnum]
1978 if [p for p in self.cloneExclude
1979 if p4PathStartsWith(path, p)]:
1980 found = False
1981 else:
1982 found = [p for p in self.depotPaths
1983 if p4PathStartsWith(path, p)]
1984 if not found:
1985 fnum = fnum + 1
1986 continue
1988 file = {}
1989 file["path"] = path
1990 file["rev"] = commit["rev%s" % fnum]
1991 file["action"] = commit["action%s" % fnum]
1992 file["type"] = commit["type%s" % fnum]
1993 files.append(file)
1994 fnum = fnum + 1
1995 return files
1997 def stripRepoPath(self, path, prefixes):
1998 """When streaming files, this is called to map a p4 depot path
1999 to where it should go in git. The prefixes are either
2000 self.depotPaths, or self.branchPrefixes in the case of
2001 branch detection."""
2003 if self.useClientSpec:
2004 # branch detection moves files up a level (the branch name)
2005 # from what client spec interpretation gives
2006 path = self.clientSpecDirs.map_in_client(path)
2007 if self.detectBranches:
2008 for b in self.knownBranches:
2009 if path.startswith(b + "/"):
2010 path = path[len(b)+1:]
2012 elif self.keepRepoPath:
2013 # Preserve everything in relative path name except leading
2014 # //depot/; just look at first prefix as they all should
2015 # be in the same depot.
2016 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2017 if p4PathStartsWith(path, depot):
2018 path = path[len(depot):]
2020 else:
2021 for p in prefixes:
2022 if p4PathStartsWith(path, p):
2023 path = path[len(p):]
2024 break
2026 path = wildcard_decode(path)
2027 return path
2029 def splitFilesIntoBranches(self, commit):
2030 """Look at each depotFile in the commit to figure out to what
2031 branch it belongs."""
2033 branches = {}
2034 fnum = 0
2035 while commit.has_key("depotFile%s" % fnum):
2036 path = commit["depotFile%s" % fnum]
2037 found = [p for p in self.depotPaths
2038 if p4PathStartsWith(path, p)]
2039 if not found:
2040 fnum = fnum + 1
2041 continue
2043 file = {}
2044 file["path"] = path
2045 file["rev"] = commit["rev%s" % fnum]
2046 file["action"] = commit["action%s" % fnum]
2047 file["type"] = commit["type%s" % fnum]
2048 fnum = fnum + 1
2050 # start with the full relative path where this file would
2051 # go in a p4 client
2052 if self.useClientSpec:
2053 relPath = self.clientSpecDirs.map_in_client(path)
2054 else:
2055 relPath = self.stripRepoPath(path, self.depotPaths)
2057 for branch in self.knownBranches.keys():
2058 # add a trailing slash so that a commit into qt/4.2foo
2059 # doesn't end up in qt/4.2, e.g.
2060 if relPath.startswith(branch + "/"):
2061 if branch not in branches:
2062 branches[branch] = []
2063 branches[branch].append(file)
2064 break
2066 return branches
2068 # output one file from the P4 stream
2069 # - helper for streamP4Files
2071 def streamOneP4File(self, file, contents):
2072 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2073 if verbose:
2074 sys.stderr.write("%s\n" % relPath)
2076 (type_base, type_mods) = split_p4_type(file["type"])
2078 git_mode = "100644"
2079 if "x" in type_mods:
2080 git_mode = "100755"
2081 if type_base == "symlink":
2082 git_mode = "120000"
2083 # p4 print on a symlink contains "target\n"; remove the newline
2084 data = ''.join(contents)
2085 contents = [data[:-1]]
2087 if type_base == "utf16":
2088 # p4 delivers different text in the python output to -G
2089 # than it does when using "print -o", or normal p4 client
2090 # operations. utf16 is converted to ascii or utf8, perhaps.
2091 # But ascii text saved as -t utf16 is completely mangled.
2092 # Invoke print -o to get the real contents.
2093 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2094 contents = [ text ]
2096 if type_base == "apple":
2097 # Apple filetype files will be streamed as a concatenation of
2098 # its appledouble header and the contents. This is useless
2099 # on both macs and non-macs. If using "print -q -o xx", it
2100 # will create "xx" with the data, and "%xx" with the header.
2101 # This is also not very useful.
2103 # Ideally, someday, this script can learn how to generate
2104 # appledouble files directly and import those to git, but
2105 # non-mac machines can never find a use for apple filetype.
2106 print "\nIgnoring apple filetype file %s" % file['depotFile']
2107 return
2109 # Perhaps windows wants unicode, utf16 newlines translated too;
2110 # but this is not doing it.
2111 if self.isWindows and type_base == "text":
2112 mangled = []
2113 for data in contents:
2114 data = data.replace("\r\n", "\n")
2115 mangled.append(data)
2116 contents = mangled
2118 # Note that we do not try to de-mangle keywords on utf16 files,
2119 # even though in theory somebody may want that.
2120 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2121 if pattern:
2122 regexp = re.compile(pattern, re.VERBOSE)
2123 text = ''.join(contents)
2124 text = regexp.sub(r'$\1$', text)
2125 contents = [ text ]
2127 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2129 # total length...
2130 length = 0
2131 for d in contents:
2132 length = length + len(d)
2134 self.gitStream.write("data %d\n" % length)
2135 for d in contents:
2136 self.gitStream.write(d)
2137 self.gitStream.write("\n")
2139 def streamOneP4Deletion(self, file):
2140 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2141 if verbose:
2142 sys.stderr.write("delete %s\n" % relPath)
2143 self.gitStream.write("D %s\n" % relPath)
2145 # handle another chunk of streaming data
2146 def streamP4FilesCb(self, marshalled):
2148 # catch p4 errors and complain
2149 err = None
2150 if "code" in marshalled:
2151 if marshalled["code"] == "error":
2152 if "data" in marshalled:
2153 err = marshalled["data"].rstrip()
2154 if err:
2155 f = None
2156 if self.stream_have_file_info:
2157 if "depotFile" in self.stream_file:
2158 f = self.stream_file["depotFile"]
2159 # force a failure in fast-import, else an empty
2160 # commit will be made
2161 self.gitStream.write("\n")
2162 self.gitStream.write("die-now\n")
2163 self.gitStream.close()
2164 # ignore errors, but make sure it exits first
2165 self.importProcess.wait()
2166 if f:
2167 die("Error from p4 print for %s: %s" % (f, err))
2168 else:
2169 die("Error from p4 print: %s" % err)
2171 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2172 # start of a new file - output the old one first
2173 self.streamOneP4File(self.stream_file, self.stream_contents)
2174 self.stream_file = {}
2175 self.stream_contents = []
2176 self.stream_have_file_info = False
2178 # pick up the new file information... for the
2179 # 'data' field we need to append to our array
2180 for k in marshalled.keys():
2181 if k == 'data':
2182 self.stream_contents.append(marshalled['data'])
2183 else:
2184 self.stream_file[k] = marshalled[k]
2186 self.stream_have_file_info = True
2188 # Stream directly from "p4 files" into "git fast-import"
2189 def streamP4Files(self, files):
2190 filesForCommit = []
2191 filesToRead = []
2192 filesToDelete = []
2194 for f in files:
2195 # if using a client spec, only add the files that have
2196 # a path in the client
2197 if self.clientSpecDirs:
2198 if self.clientSpecDirs.map_in_client(f['path']) == "":
2199 continue
2201 filesForCommit.append(f)
2202 if f['action'] in self.delete_actions:
2203 filesToDelete.append(f)
2204 else:
2205 filesToRead.append(f)
2207 # deleted files...
2208 for f in filesToDelete:
2209 self.streamOneP4Deletion(f)
2211 if len(filesToRead) > 0:
2212 self.stream_file = {}
2213 self.stream_contents = []
2214 self.stream_have_file_info = False
2216 # curry self argument
2217 def streamP4FilesCbSelf(entry):
2218 self.streamP4FilesCb(entry)
2220 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2222 p4CmdList(["-x", "-", "print"],
2223 stdin=fileArgs,
2224 cb=streamP4FilesCbSelf)
2226 # do the last chunk
2227 if self.stream_file.has_key('depotFile'):
2228 self.streamOneP4File(self.stream_file, self.stream_contents)
2230 def make_email(self, userid):
2231 if userid in self.users:
2232 return self.users[userid]
2233 else:
2234 return "%s <a@b>" % userid
2236 # Stream a p4 tag
2237 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2238 if verbose:
2239 print "writing tag %s for commit %s" % (labelName, commit)
2240 gitStream.write("tag %s\n" % labelName)
2241 gitStream.write("from %s\n" % commit)
2243 if labelDetails.has_key('Owner'):
2244 owner = labelDetails["Owner"]
2245 else:
2246 owner = None
2248 # Try to use the owner of the p4 label, or failing that,
2249 # the current p4 user id.
2250 if owner:
2251 email = self.make_email(owner)
2252 else:
2253 email = self.make_email(self.p4UserId())
2254 tagger = "%s %s %s" % (email, epoch, self.tz)
2256 gitStream.write("tagger %s\n" % tagger)
2258 print "labelDetails=",labelDetails
2259 if labelDetails.has_key('Description'):
2260 description = labelDetails['Description']
2261 else:
2262 description = 'Label from git p4'
2264 gitStream.write("data %d\n" % len(description))
2265 gitStream.write(description)
2266 gitStream.write("\n")
2268 def commit(self, details, files, branch, parent = ""):
2269 epoch = details["time"]
2270 author = details["user"]
2272 if self.verbose:
2273 print "commit into %s" % branch
2275 # start with reading files; if that fails, we should not
2276 # create a commit.
2277 new_files = []
2278 for f in files:
2279 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2280 new_files.append (f)
2281 else:
2282 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2284 self.gitStream.write("commit %s\n" % branch)
2285 # gitStream.write("mark :%s\n" % details["change"])
2286 self.committedChanges.add(int(details["change"]))
2287 committer = ""
2288 if author not in self.users:
2289 self.getUserMapFromPerforceServer()
2290 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2292 self.gitStream.write("committer %s\n" % committer)
2294 self.gitStream.write("data <<EOT\n")
2295 self.gitStream.write(details["desc"])
2296 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2297 (','.join(self.branchPrefixes), details["change"]))
2298 if len(details['options']) > 0:
2299 self.gitStream.write(": options = %s" % details['options'])
2300 self.gitStream.write("]\nEOT\n\n")
2302 if len(parent) > 0:
2303 if self.verbose:
2304 print "parent %s" % parent
2305 self.gitStream.write("from %s\n" % parent)
2307 self.streamP4Files(new_files)
2308 self.gitStream.write("\n")
2310 change = int(details["change"])
2312 if self.labels.has_key(change):
2313 label = self.labels[change]
2314 labelDetails = label[0]
2315 labelRevisions = label[1]
2316 if self.verbose:
2317 print "Change %s is labelled %s" % (change, labelDetails)
2319 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2320 for p in self.branchPrefixes])
2322 if len(files) == len(labelRevisions):
2324 cleanedFiles = {}
2325 for info in files:
2326 if info["action"] in self.delete_actions:
2327 continue
2328 cleanedFiles[info["depotFile"]] = info["rev"]
2330 if cleanedFiles == labelRevisions:
2331 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2333 else:
2334 if not self.silent:
2335 print ("Tag %s does not match with change %s: files do not match."
2336 % (labelDetails["label"], change))
2338 else:
2339 if not self.silent:
2340 print ("Tag %s does not match with change %s: file count is different."
2341 % (labelDetails["label"], change))
2343 # Build a dictionary of changelists and labels, for "detect-labels" option.
2344 def getLabels(self):
2345 self.labels = {}
2347 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2348 if len(l) > 0 and not self.silent:
2349 print "Finding files belonging to labels in %s" % `self.depotPaths`
2351 for output in l:
2352 label = output["label"]
2353 revisions = {}
2354 newestChange = 0
2355 if self.verbose:
2356 print "Querying files for label %s" % label
2357 for file in p4CmdList(["files"] +
2358 ["%s...@%s" % (p, label)
2359 for p in self.depotPaths]):
2360 revisions[file["depotFile"]] = file["rev"]
2361 change = int(file["change"])
2362 if change > newestChange:
2363 newestChange = change
2365 self.labels[newestChange] = [output, revisions]
2367 if self.verbose:
2368 print "Label changes: %s" % self.labels.keys()
2370 # Import p4 labels as git tags. A direct mapping does not
2371 # exist, so assume that if all the files are at the same revision
2372 # then we can use that, or it's something more complicated we should
2373 # just ignore.
2374 def importP4Labels(self, stream, p4Labels):
2375 if verbose:
2376 print "import p4 labels: " + ' '.join(p4Labels)
2378 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2379 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2380 if len(validLabelRegexp) == 0:
2381 validLabelRegexp = defaultLabelRegexp
2382 m = re.compile(validLabelRegexp)
2384 for name in p4Labels:
2385 commitFound = False
2387 if not m.match(name):
2388 if verbose:
2389 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2390 continue
2392 if name in ignoredP4Labels:
2393 continue
2395 labelDetails = p4CmdList(['label', "-o", name])[0]
2397 # get the most recent changelist for each file in this label
2398 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2399 for p in self.depotPaths])
2401 if change.has_key('change'):
2402 # find the corresponding git commit; take the oldest commit
2403 changelist = int(change['change'])
2404 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2405 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2406 if len(gitCommit) == 0:
2407 print "could not find git commit for changelist %d" % changelist
2408 else:
2409 gitCommit = gitCommit.strip()
2410 commitFound = True
2411 # Convert from p4 time format
2412 try:
2413 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2414 except ValueError:
2415 print "Could not convert label time %s" % labelDetails['Update']
2416 tmwhen = 1
2418 when = int(time.mktime(tmwhen))
2419 self.streamTag(stream, name, labelDetails, gitCommit, when)
2420 if verbose:
2421 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2422 else:
2423 if verbose:
2424 print "Label %s has no changelists - possibly deleted?" % name
2426 if not commitFound:
2427 # We can't import this label; don't try again as it will get very
2428 # expensive repeatedly fetching all the files for labels that will
2429 # never be imported. If the label is moved in the future, the
2430 # ignore will need to be removed manually.
2431 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2433 def guessProjectName(self):
2434 for p in self.depotPaths:
2435 if p.endswith("/"):
2436 p = p[:-1]
2437 p = p[p.strip().rfind("/") + 1:]
2438 if not p.endswith("/"):
2439 p += "/"
2440 return p
2442 def getBranchMapping(self):
2443 lostAndFoundBranches = set()
2445 user = gitConfig("git-p4.branchUser")
2446 if len(user) > 0:
2447 command = "branches -u %s" % user
2448 else:
2449 command = "branches"
2451 for info in p4CmdList(command):
2452 details = p4Cmd(["branch", "-o", info["branch"]])
2453 viewIdx = 0
2454 while details.has_key("View%s" % viewIdx):
2455 paths = details["View%s" % viewIdx].split(" ")
2456 viewIdx = viewIdx + 1
2457 # require standard //depot/foo/... //depot/bar/... mapping
2458 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2459 continue
2460 source = paths[0]
2461 destination = paths[1]
2462 ## HACK
2463 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2464 source = source[len(self.depotPaths[0]):-4]
2465 destination = destination[len(self.depotPaths[0]):-4]
2467 if destination in self.knownBranches:
2468 if not self.silent:
2469 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2470 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2471 continue
2473 self.knownBranches[destination] = source
2475 lostAndFoundBranches.discard(destination)
2477 if source not in self.knownBranches:
2478 lostAndFoundBranches.add(source)
2480 # Perforce does not strictly require branches to be defined, so we also
2481 # check git config for a branch list.
2483 # Example of branch definition in git config file:
2484 # [git-p4]
2485 # branchList=main:branchA
2486 # branchList=main:branchB
2487 # branchList=branchA:branchC
2488 configBranches = gitConfigList("git-p4.branchList")
2489 for branch in configBranches:
2490 if branch:
2491 (source, destination) = branch.split(":")
2492 self.knownBranches[destination] = source
2494 lostAndFoundBranches.discard(destination)
2496 if source not in self.knownBranches:
2497 lostAndFoundBranches.add(source)
2500 for branch in lostAndFoundBranches:
2501 self.knownBranches[branch] = branch
2503 def getBranchMappingFromGitBranches(self):
2504 branches = p4BranchesInGit(self.importIntoRemotes)
2505 for branch in branches.keys():
2506 if branch == "master":
2507 branch = "main"
2508 else:
2509 branch = branch[len(self.projectName):]
2510 self.knownBranches[branch] = branch
2512 def listExistingP4GitBranches(self):
2513 # branches holds mapping from name to commit
2514 branches = p4BranchesInGit(self.importIntoRemotes)
2515 self.p4BranchesInGit = branches.keys()
2516 for branch in branches.keys():
2517 self.initialParents[self.refPrefix + branch] = branches[branch]
2519 def updateOptionDict(self, d):
2520 option_keys = {}
2521 if self.keepRepoPath:
2522 option_keys['keepRepoPath'] = 1
2524 d["options"] = ' '.join(sorted(option_keys.keys()))
2526 def readOptions(self, d):
2527 self.keepRepoPath = (d.has_key('options')
2528 and ('keepRepoPath' in d['options']))
2530 def gitRefForBranch(self, branch):
2531 if branch == "main":
2532 return self.refPrefix + "master"
2534 if len(branch) <= 0:
2535 return branch
2537 return self.refPrefix + self.projectName + branch
2539 def gitCommitByP4Change(self, ref, change):
2540 if self.verbose:
2541 print "looking in ref " + ref + " for change %s using bisect..." % change
2543 earliestCommit = ""
2544 latestCommit = parseRevision(ref)
2546 while True:
2547 if self.verbose:
2548 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2549 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2550 if len(next) == 0:
2551 if self.verbose:
2552 print "argh"
2553 return ""
2554 log = extractLogMessageFromGitCommit(next)
2555 settings = extractSettingsGitLog(log)
2556 currentChange = int(settings['change'])
2557 if self.verbose:
2558 print "current change %s" % currentChange
2560 if currentChange == change:
2561 if self.verbose:
2562 print "found %s" % next
2563 return next
2565 if currentChange < change:
2566 earliestCommit = "^%s" % next
2567 else:
2568 latestCommit = "%s" % next
2570 return ""
2572 def importNewBranch(self, branch, maxChange):
2573 # make fast-import flush all changes to disk and update the refs using the checkpoint
2574 # command so that we can try to find the branch parent in the git history
2575 self.gitStream.write("checkpoint\n\n");
2576 self.gitStream.flush();
2577 branchPrefix = self.depotPaths[0] + branch + "/"
2578 range = "@1,%s" % maxChange
2579 #print "prefix" + branchPrefix
2580 changes = p4ChangesForPaths([branchPrefix], range)
2581 if len(changes) <= 0:
2582 return False
2583 firstChange = changes[0]
2584 #print "first change in branch: %s" % firstChange
2585 sourceBranch = self.knownBranches[branch]
2586 sourceDepotPath = self.depotPaths[0] + sourceBranch
2587 sourceRef = self.gitRefForBranch(sourceBranch)
2588 #print "source " + sourceBranch
2590 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2591 #print "branch parent: %s" % branchParentChange
2592 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2593 if len(gitParent) > 0:
2594 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2595 #print "parent git commit: %s" % gitParent
2597 self.importChanges(changes)
2598 return True
2600 def searchParent(self, parent, branch, target):
2601 parentFound = False
2602 for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2603 blob = blob.strip()
2604 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2605 parentFound = True
2606 if self.verbose:
2607 print "Found parent of %s in commit %s" % (branch, blob)
2608 break
2609 if parentFound:
2610 return blob
2611 else:
2612 return None
2614 def importChanges(self, changes):
2615 cnt = 1
2616 for change in changes:
2617 description = p4_describe(change)
2618 self.updateOptionDict(description)
2620 if not self.silent:
2621 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2622 sys.stdout.flush()
2623 cnt = cnt + 1
2625 try:
2626 if self.detectBranches:
2627 branches = self.splitFilesIntoBranches(description)
2628 for branch in branches.keys():
2629 ## HACK --hwn
2630 branchPrefix = self.depotPaths[0] + branch + "/"
2631 self.branchPrefixes = [ branchPrefix ]
2633 parent = ""
2635 filesForCommit = branches[branch]
2637 if self.verbose:
2638 print "branch is %s" % branch
2640 self.updatedBranches.add(branch)
2642 if branch not in self.createdBranches:
2643 self.createdBranches.add(branch)
2644 parent = self.knownBranches[branch]
2645 if parent == branch:
2646 parent = ""
2647 else:
2648 fullBranch = self.projectName + branch
2649 if fullBranch not in self.p4BranchesInGit:
2650 if not self.silent:
2651 print("\n Importing new branch %s" % fullBranch);
2652 if self.importNewBranch(branch, change - 1):
2653 parent = ""
2654 self.p4BranchesInGit.append(fullBranch)
2655 if not self.silent:
2656 print("\n Resuming with change %s" % change);
2658 if self.verbose:
2659 print "parent determined through known branches: %s" % parent
2661 branch = self.gitRefForBranch(branch)
2662 parent = self.gitRefForBranch(parent)
2664 if self.verbose:
2665 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2667 if len(parent) == 0 and branch in self.initialParents:
2668 parent = self.initialParents[branch]
2669 del self.initialParents[branch]
2671 blob = None
2672 if len(parent) > 0:
2673 tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2674 if self.verbose:
2675 print "Creating temporary branch: " + tempBranch
2676 self.commit(description, filesForCommit, tempBranch)
2677 self.tempBranches.append(tempBranch)
2678 self.checkpoint()
2679 blob = self.searchParent(parent, branch, tempBranch)
2680 if blob:
2681 self.commit(description, filesForCommit, branch, blob)
2682 else:
2683 if self.verbose:
2684 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2685 self.commit(description, filesForCommit, branch, parent)
2686 else:
2687 files = self.extractFilesFromCommit(description)
2688 self.commit(description, files, self.branch,
2689 self.initialParent)
2690 self.initialParent = ""
2691 except IOError:
2692 print self.gitError.read()
2693 sys.exit(1)
2695 def importHeadRevision(self, revision):
2696 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2698 details = {}
2699 details["user"] = "git perforce import user"
2700 details["desc"] = ("Initial import of %s from the state at revision %s\n"
2701 % (' '.join(self.depotPaths), revision))
2702 details["change"] = revision
2703 newestRevision = 0
2705 fileCnt = 0
2706 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2708 for info in p4CmdList(["files"] + fileArgs):
2710 if 'code' in info and info['code'] == 'error':
2711 sys.stderr.write("p4 returned an error: %s\n"
2712 % info['data'])
2713 if info['data'].find("must refer to client") >= 0:
2714 sys.stderr.write("This particular p4 error is misleading.\n")
2715 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2716 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
2717 sys.exit(1)
2718 if 'p4ExitCode' in info:
2719 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2720 sys.exit(1)
2723 change = int(info["change"])
2724 if change > newestRevision:
2725 newestRevision = change
2727 if info["action"] in self.delete_actions:
2728 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2729 #fileCnt = fileCnt + 1
2730 continue
2732 for prop in ["depotFile", "rev", "action", "type" ]:
2733 details["%s%s" % (prop, fileCnt)] = info[prop]
2735 fileCnt = fileCnt + 1
2737 details["change"] = newestRevision
2739 # Use time from top-most change so that all git p4 clones of
2740 # the same p4 repo have the same commit SHA1s.
2741 res = p4_describe(newestRevision)
2742 details["time"] = res["time"]
2744 self.updateOptionDict(details)
2745 try:
2746 self.commit(details, self.extractFilesFromCommit(details), self.branch)
2747 except IOError:
2748 print "IO error with git fast-import. Is your git version recent enough?"
2749 print self.gitError.read()
2752 def run(self, args):
2753 self.depotPaths = []
2754 self.changeRange = ""
2755 self.initialParent = ""
2756 self.previousDepotPaths = []
2758 # map from branch depot path to parent branch
2759 self.knownBranches = {}
2760 self.initialParents = {}
2761 self.hasOrigin = originP4BranchesExist()
2762 if not self.syncWithOrigin:
2763 self.hasOrigin = False
2765 if self.importIntoRemotes:
2766 self.refPrefix = "refs/remotes/p4/"
2767 else:
2768 self.refPrefix = "refs/heads/p4/"
2770 if self.syncWithOrigin and self.hasOrigin:
2771 if not self.silent:
2772 print "Syncing with origin first by calling git fetch origin"
2773 system("git fetch origin")
2775 if len(self.branch) == 0:
2776 self.branch = self.refPrefix + "master"
2777 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2778 system("git update-ref %s refs/heads/p4" % self.branch)
2779 system("git branch -D p4");
2780 # create it /after/ importing, when master exists
2781 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2782 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2784 # accept either the command-line option, or the configuration variable
2785 if self.useClientSpec:
2786 # will use this after clone to set the variable
2787 self.useClientSpec_from_options = True
2788 else:
2789 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2790 self.useClientSpec = True
2791 if self.useClientSpec:
2792 self.clientSpecDirs = getClientSpec()
2794 # TODO: should always look at previous commits,
2795 # merge with previous imports, if possible.
2796 if args == []:
2797 if self.hasOrigin:
2798 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2799 self.listExistingP4GitBranches()
2801 if len(self.p4BranchesInGit) > 1:
2802 if not self.silent:
2803 print "Importing from/into multiple branches"
2804 self.detectBranches = True
2806 if self.verbose:
2807 print "branches: %s" % self.p4BranchesInGit
2809 p4Change = 0
2810 for branch in self.p4BranchesInGit:
2811 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
2813 settings = extractSettingsGitLog(logMsg)
2815 self.readOptions(settings)
2816 if (settings.has_key('depot-paths')
2817 and settings.has_key ('change')):
2818 change = int(settings['change']) + 1
2819 p4Change = max(p4Change, change)
2821 depotPaths = sorted(settings['depot-paths'])
2822 if self.previousDepotPaths == []:
2823 self.previousDepotPaths = depotPaths
2824 else:
2825 paths = []
2826 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2827 prev_list = prev.split("/")
2828 cur_list = cur.split("/")
2829 for i in range(0, min(len(cur_list), len(prev_list))):
2830 if cur_list[i] <> prev_list[i]:
2831 i = i - 1
2832 break
2834 paths.append ("/".join(cur_list[:i + 1]))
2836 self.previousDepotPaths = paths
2838 if p4Change > 0:
2839 self.depotPaths = sorted(self.previousDepotPaths)
2840 self.changeRange = "@%s,#head" % p4Change
2841 if not self.detectBranches:
2842 self.initialParent = parseRevision(self.branch)
2843 if not self.silent and not self.detectBranches:
2844 print "Performing incremental import into %s git branch" % self.branch
2846 if not self.branch.startswith("refs/"):
2847 self.branch = "refs/heads/" + self.branch
2849 if len(args) == 0 and self.depotPaths:
2850 if not self.silent:
2851 print "Depot paths: %s" % ' '.join(self.depotPaths)
2852 else:
2853 if self.depotPaths and self.depotPaths != args:
2854 print ("previous import used depot path %s and now %s was specified. "
2855 "This doesn't work!" % (' '.join (self.depotPaths),
2856 ' '.join (args)))
2857 sys.exit(1)
2859 self.depotPaths = sorted(args)
2861 revision = ""
2862 self.users = {}
2864 # Make sure no revision specifiers are used when --changesfile
2865 # is specified.
2866 bad_changesfile = False
2867 if len(self.changesFile) > 0:
2868 for p in self.depotPaths:
2869 if p.find("@") >= 0 or p.find("#") >= 0:
2870 bad_changesfile = True
2871 break
2872 if bad_changesfile:
2873 die("Option --changesfile is incompatible with revision specifiers")
2875 newPaths = []
2876 for p in self.depotPaths:
2877 if p.find("@") != -1:
2878 atIdx = p.index("@")
2879 self.changeRange = p[atIdx:]
2880 if self.changeRange == "@all":
2881 self.changeRange = ""
2882 elif ',' not in self.changeRange:
2883 revision = self.changeRange
2884 self.changeRange = ""
2885 p = p[:atIdx]
2886 elif p.find("#") != -1:
2887 hashIdx = p.index("#")
2888 revision = p[hashIdx:]
2889 p = p[:hashIdx]
2890 elif self.previousDepotPaths == []:
2891 # pay attention to changesfile, if given, else import
2892 # the entire p4 tree at the head revision
2893 if len(self.changesFile) == 0:
2894 revision = "#head"
2896 p = re.sub ("\.\.\.$", "", p)
2897 if not p.endswith("/"):
2898 p += "/"
2900 newPaths.append(p)
2902 self.depotPaths = newPaths
2904 # --detect-branches may change this for each branch
2905 self.branchPrefixes = self.depotPaths
2907 self.loadUserMapFromCache()
2908 self.labels = {}
2909 if self.detectLabels:
2910 self.getLabels();
2912 if self.detectBranches:
2913 ## FIXME - what's a P4 projectName ?
2914 self.projectName = self.guessProjectName()
2916 if self.hasOrigin:
2917 self.getBranchMappingFromGitBranches()
2918 else:
2919 self.getBranchMapping()
2920 if self.verbose:
2921 print "p4-git branches: %s" % self.p4BranchesInGit
2922 print "initial parents: %s" % self.initialParents
2923 for b in self.p4BranchesInGit:
2924 if b != "master":
2926 ## FIXME
2927 b = b[len(self.projectName):]
2928 self.createdBranches.add(b)
2930 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2932 self.importProcess = subprocess.Popen(["git", "fast-import"],
2933 stdin=subprocess.PIPE,
2934 stdout=subprocess.PIPE,
2935 stderr=subprocess.PIPE);
2936 self.gitOutput = self.importProcess.stdout
2937 self.gitStream = self.importProcess.stdin
2938 self.gitError = self.importProcess.stderr
2940 if revision:
2941 self.importHeadRevision(revision)
2942 else:
2943 changes = []
2945 if len(self.changesFile) > 0:
2946 output = open(self.changesFile).readlines()
2947 changeSet = set()
2948 for line in output:
2949 changeSet.add(int(line))
2951 for change in changeSet:
2952 changes.append(change)
2954 changes.sort()
2955 else:
2956 # catch "git p4 sync" with no new branches, in a repo that
2957 # does not have any existing p4 branches
2958 if len(args) == 0 and not self.p4BranchesInGit:
2959 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2960 if self.verbose:
2961 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2962 self.changeRange)
2963 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2965 if len(self.maxChanges) > 0:
2966 changes = changes[:min(int(self.maxChanges), len(changes))]
2968 if len(changes) == 0:
2969 if not self.silent:
2970 print "No changes to import!"
2971 else:
2972 if not self.silent and not self.detectBranches:
2973 print "Import destination: %s" % self.branch
2975 self.updatedBranches = set()
2977 self.importChanges(changes)
2979 if not self.silent:
2980 print ""
2981 if len(self.updatedBranches) > 0:
2982 sys.stdout.write("Updated branches: ")
2983 for b in self.updatedBranches:
2984 sys.stdout.write("%s " % b)
2985 sys.stdout.write("\n")
2987 if gitConfig("git-p4.importLabels", "--bool") == "true":
2988 self.importLabels = True
2990 if self.importLabels:
2991 p4Labels = getP4Labels(self.depotPaths)
2992 gitTags = getGitTags()
2994 missingP4Labels = p4Labels - gitTags
2995 self.importP4Labels(self.gitStream, missingP4Labels)
2997 self.gitStream.close()
2998 if self.importProcess.wait() != 0:
2999 die("fast-import failed: %s" % self.gitError.read())
3000 self.gitOutput.close()
3001 self.gitError.close()
3003 # Cleanup temporary branches created during import
3004 if self.tempBranches != []:
3005 for branch in self.tempBranches:
3006 read_pipe("git update-ref -d %s" % branch)
3007 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3009 return True
3011 class P4Rebase(Command):
3012 def __init__(self):
3013 Command.__init__(self)
3014 self.options = [
3015 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3017 self.importLabels = False
3018 self.description = ("Fetches the latest revision from perforce and "
3019 + "rebases the current work (branch) against it")
3021 def run(self, args):
3022 sync = P4Sync()
3023 sync.importLabels = self.importLabels
3024 sync.run([])
3026 return self.rebase()
3028 def rebase(self):
3029 if os.system("git update-index --refresh") != 0:
3030 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3031 if len(read_pipe("git diff-index HEAD --")) > 0:
3032 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
3034 [upstream, settings] = findUpstreamBranchPoint()
3035 if len(upstream) == 0:
3036 die("Cannot find upstream branchpoint for rebase")
3038 # the branchpoint may be p4/foo~3, so strip off the parent
3039 upstream = re.sub("~[0-9]+$", "", upstream)
3041 print "Rebasing the current branch onto %s" % upstream
3042 oldHead = read_pipe("git rev-parse HEAD").strip()
3043 system("git rebase %s" % upstream)
3044 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3045 return True
3047 class P4Clone(P4Sync):
3048 def __init__(self):
3049 P4Sync.__init__(self)
3050 self.description = "Creates a new git repository and imports from Perforce into it"
3051 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3052 self.options += [
3053 optparse.make_option("--destination", dest="cloneDestination",
3054 action='store', default=None,
3055 help="where to leave result of the clone"),
3056 optparse.make_option("-/", dest="cloneExclude",
3057 action="append", type="string",
3058 help="exclude depot path"),
3059 optparse.make_option("--bare", dest="cloneBare",
3060 action="store_true", default=False),
3062 self.cloneDestination = None
3063 self.needsGit = False
3064 self.cloneBare = False
3066 # This is required for the "append" cloneExclude action
3067 def ensure_value(self, attr, value):
3068 if not hasattr(self, attr) or getattr(self, attr) is None:
3069 setattr(self, attr, value)
3070 return getattr(self, attr)
3072 def defaultDestination(self, args):
3073 ## TODO: use common prefix of args?
3074 depotPath = args[0]
3075 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3076 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3077 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3078 depotDir = re.sub(r"/$", "", depotDir)
3079 return os.path.split(depotDir)[1]
3081 def run(self, args):
3082 if len(args) < 1:
3083 return False
3085 if self.keepRepoPath and not self.cloneDestination:
3086 sys.stderr.write("Must specify destination for --keep-path\n")
3087 sys.exit(1)
3089 depotPaths = args
3091 if not self.cloneDestination and len(depotPaths) > 1:
3092 self.cloneDestination = depotPaths[-1]
3093 depotPaths = depotPaths[:-1]
3095 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3096 for p in depotPaths:
3097 if not p.startswith("//"):
3098 return False
3100 if not self.cloneDestination:
3101 self.cloneDestination = self.defaultDestination(args)
3103 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3105 if not os.path.exists(self.cloneDestination):
3106 os.makedirs(self.cloneDestination)
3107 chdir(self.cloneDestination)
3109 init_cmd = [ "git", "init" ]
3110 if self.cloneBare:
3111 init_cmd.append("--bare")
3112 subprocess.check_call(init_cmd)
3114 if not P4Sync.run(self, depotPaths):
3115 return False
3116 if self.branch != "master":
3117 if self.importIntoRemotes:
3118 masterbranch = "refs/remotes/p4/master"
3119 else:
3120 masterbranch = "refs/heads/p4/master"
3121 if gitBranchExists(masterbranch):
3122 system("git branch master %s" % masterbranch)
3123 if not self.cloneBare:
3124 system("git checkout -f")
3125 else:
3126 print "Could not detect main branch. No checkout/master branch created."
3128 # auto-set this variable if invoked with --use-client-spec
3129 if self.useClientSpec_from_options:
3130 system("git config --bool git-p4.useclientspec true")
3132 return True
3134 class P4Branches(Command):
3135 def __init__(self):
3136 Command.__init__(self)
3137 self.options = [ ]
3138 self.description = ("Shows the git branches that hold imports and their "
3139 + "corresponding perforce depot paths")
3140 self.verbose = False
3142 def run(self, args):
3143 if originP4BranchesExist():
3144 createOrUpdateBranchesFromOrigin()
3146 cmdline = "git rev-parse --symbolic "
3147 cmdline += " --remotes"
3149 for line in read_pipe_lines(cmdline):
3150 line = line.strip()
3152 if not line.startswith('p4/') or line == "p4/HEAD":
3153 continue
3154 branch = line
3156 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3157 settings = extractSettingsGitLog(log)
3159 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3160 return True
3162 class HelpFormatter(optparse.IndentedHelpFormatter):
3163 def __init__(self):
3164 optparse.IndentedHelpFormatter.__init__(self)
3166 def format_description(self, description):
3167 if description:
3168 return description + "\n"
3169 else:
3170 return ""
3172 def printUsage(commands):
3173 print "usage: %s <command> [options]" % sys.argv[0]
3174 print ""
3175 print "valid commands: %s" % ", ".join(commands)
3176 print ""
3177 print "Try %s <command> --help for command specific help." % sys.argv[0]
3178 print ""
3180 commands = {
3181 "debug" : P4Debug,
3182 "submit" : P4Submit,
3183 "commit" : P4Submit,
3184 "sync" : P4Sync,
3185 "rebase" : P4Rebase,
3186 "clone" : P4Clone,
3187 "rollback" : P4RollBack,
3188 "branches" : P4Branches
3192 def main():
3193 if len(sys.argv[1:]) == 0:
3194 printUsage(commands.keys())
3195 sys.exit(2)
3197 cmdName = sys.argv[1]
3198 try:
3199 klass = commands[cmdName]
3200 cmd = klass()
3201 except KeyError:
3202 print "unknown command %s" % cmdName
3203 print ""
3204 printUsage(commands.keys())
3205 sys.exit(2)
3207 options = cmd.options
3208 cmd.gitdir = os.environ.get("GIT_DIR", None)
3210 args = sys.argv[2:]
3212 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3213 if cmd.needsGit:
3214 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3216 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3217 options,
3218 description = cmd.description,
3219 formatter = HelpFormatter())
3221 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3222 global verbose
3223 verbose = cmd.verbose
3224 if cmd.needsGit:
3225 if cmd.gitdir == None:
3226 cmd.gitdir = os.path.abspath(".git")
3227 if not isValidGitDir(cmd.gitdir):
3228 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3229 if os.path.exists(cmd.gitdir):
3230 cdup = read_pipe("git rev-parse --show-cdup").strip()
3231 if len(cdup) > 0:
3232 chdir(cdup);
3234 if not isValidGitDir(cmd.gitdir):
3235 if isValidGitDir(cmd.gitdir + "/.git"):
3236 cmd.gitdir += "/.git"
3237 else:
3238 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3240 os.environ["GIT_DIR"] = cmd.gitdir
3242 if not cmd.run(args):
3243 parser.print_help()
3244 sys.exit(2)
3247 if __name__ == '__main__':
3248 main()