Merge branch 'wk/man-deny-current-branch-is-default-these-days' into maint
[git/jnareb-git.git] / git-p4.py
blob0682e61e90dca3f88f399c20aa19c5336274bc95
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse, sys, os, marshal, subprocess, shelve
12 import tempfile, getopt, os.path, time, platform
13 import re, shutil
15 try:
16 from subprocess import CalledProcessError
17 except ImportError:
18 # from python2.7:subprocess.py
19 # Exception classes used by this module.
20 class CalledProcessError(Exception):
21 """This exception is raised when a process run by check_call() returns
22 a non-zero exit status. The exit status will be stored in the
23 returncode attribute."""
24 def __init__(self, returncode, cmd):
25 self.returncode = returncode
26 self.cmd = cmd
27 def __str__(self):
28 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
30 verbose = False
32 # Only labels/tags matching this will be imported/exported
33 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
35 def p4_build_cmd(cmd):
36 """Build a suitable p4 command line.
38 This consolidates building and returning a p4 command line into one
39 location. It means that hooking into the environment, or other configuration
40 can be done more easily.
41 """
42 real_cmd = ["p4"]
44 user = gitConfig("git-p4.user")
45 if len(user) > 0:
46 real_cmd += ["-u",user]
48 password = gitConfig("git-p4.password")
49 if len(password) > 0:
50 real_cmd += ["-P", password]
52 port = gitConfig("git-p4.port")
53 if len(port) > 0:
54 real_cmd += ["-p", port]
56 host = gitConfig("git-p4.host")
57 if len(host) > 0:
58 real_cmd += ["-H", host]
60 client = gitConfig("git-p4.client")
61 if len(client) > 0:
62 real_cmd += ["-c", client]
65 if isinstance(cmd,basestring):
66 real_cmd = ' '.join(real_cmd) + ' ' + cmd
67 else:
68 real_cmd += cmd
69 return real_cmd
71 def chdir(dir):
72 # P4 uses the PWD environment variable rather than getcwd(). Since we're
73 # not using the shell, we have to set it ourselves. This path could
74 # be relative, so go there first, then figure out where we ended up.
75 os.chdir(dir)
76 os.environ['PWD'] = os.getcwd()
78 def die(msg):
79 if verbose:
80 raise Exception(msg)
81 else:
82 sys.stderr.write(msg + "\n")
83 sys.exit(1)
85 def write_pipe(c, stdin):
86 if verbose:
87 sys.stderr.write('Writing pipe: %s\n' % str(c))
89 expand = isinstance(c,basestring)
90 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
91 pipe = p.stdin
92 val = pipe.write(stdin)
93 pipe.close()
94 if p.wait():
95 die('Command failed: %s' % str(c))
97 return val
99 def p4_write_pipe(c, stdin):
100 real_cmd = p4_build_cmd(c)
101 return write_pipe(real_cmd, stdin)
103 def read_pipe(c, ignore_error=False):
104 if verbose:
105 sys.stderr.write('Reading pipe: %s\n' % str(c))
107 expand = isinstance(c,basestring)
108 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
109 pipe = p.stdout
110 val = pipe.read()
111 if p.wait() and not ignore_error:
112 die('Command failed: %s' % str(c))
114 return val
116 def p4_read_pipe(c, ignore_error=False):
117 real_cmd = p4_build_cmd(c)
118 return read_pipe(real_cmd, ignore_error)
120 def read_pipe_lines(c):
121 if verbose:
122 sys.stderr.write('Reading pipe: %s\n' % str(c))
124 expand = isinstance(c, basestring)
125 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
126 pipe = p.stdout
127 val = pipe.readlines()
128 if pipe.close() or p.wait():
129 die('Command failed: %s' % str(c))
131 return val
133 def p4_read_pipe_lines(c):
134 """Specifically invoke p4 on the command supplied. """
135 real_cmd = p4_build_cmd(c)
136 return read_pipe_lines(real_cmd)
138 def p4_has_command(cmd):
139 """Ask p4 for help on this command. If it returns an error, the
140 command does not exist in this version of p4."""
141 real_cmd = p4_build_cmd(["help", cmd])
142 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
143 stderr=subprocess.PIPE)
144 p.communicate()
145 return p.returncode == 0
147 def p4_has_move_command():
148 """See if the move command exists, that it supports -k, and that
149 it has not been administratively disabled. The arguments
150 must be correct, but the filenames do not have to exist. Use
151 ones with wildcards so even if they exist, it will fail."""
153 if not p4_has_command("move"):
154 return False
155 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
156 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
157 (out, err) = p.communicate()
158 # return code will be 1 in either case
159 if err.find("Invalid option") >= 0:
160 return False
161 if err.find("disabled") >= 0:
162 return False
163 # assume it failed because @... was invalid changelist
164 return True
166 def system(cmd):
167 expand = isinstance(cmd,basestring)
168 if verbose:
169 sys.stderr.write("executing %s\n" % str(cmd))
170 retcode = subprocess.call(cmd, shell=expand)
171 if retcode:
172 raise CalledProcessError(retcode, cmd)
174 def p4_system(cmd):
175 """Specifically invoke p4 as the system command. """
176 real_cmd = p4_build_cmd(cmd)
177 expand = isinstance(real_cmd, basestring)
178 retcode = subprocess.call(real_cmd, shell=expand)
179 if retcode:
180 raise CalledProcessError(retcode, real_cmd)
182 def p4_integrate(src, dest):
183 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
185 def p4_sync(f, *options):
186 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
188 def p4_add(f):
189 # forcibly add file names with wildcards
190 if wildcard_present(f):
191 p4_system(["add", "-f", f])
192 else:
193 p4_system(["add", f])
195 def p4_delete(f):
196 p4_system(["delete", wildcard_encode(f)])
198 def p4_edit(f):
199 p4_system(["edit", wildcard_encode(f)])
201 def p4_revert(f):
202 p4_system(["revert", wildcard_encode(f)])
204 def p4_reopen(type, f):
205 p4_system(["reopen", "-t", type, wildcard_encode(f)])
207 def p4_move(src, dest):
208 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
210 def p4_describe(change):
211 """Make sure it returns a valid result by checking for
212 the presence of field "time". Return a dict of the
213 results."""
215 ds = p4CmdList(["describe", "-s", str(change)])
216 if len(ds) != 1:
217 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
219 d = ds[0]
221 if "p4ExitCode" in d:
222 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
223 str(d)))
224 if "code" in d:
225 if d["code"] == "error":
226 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
228 if "time" not in d:
229 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
231 return d
234 # Canonicalize the p4 type and return a tuple of the
235 # base type, plus any modifiers. See "p4 help filetypes"
236 # for a list and explanation.
238 def split_p4_type(p4type):
240 p4_filetypes_historical = {
241 "ctempobj": "binary+Sw",
242 "ctext": "text+C",
243 "cxtext": "text+Cx",
244 "ktext": "text+k",
245 "kxtext": "text+kx",
246 "ltext": "text+F",
247 "tempobj": "binary+FSw",
248 "ubinary": "binary+F",
249 "uresource": "resource+F",
250 "uxbinary": "binary+Fx",
251 "xbinary": "binary+x",
252 "xltext": "text+Fx",
253 "xtempobj": "binary+Swx",
254 "xtext": "text+x",
255 "xunicode": "unicode+x",
256 "xutf16": "utf16+x",
258 if p4type in p4_filetypes_historical:
259 p4type = p4_filetypes_historical[p4type]
260 mods = ""
261 s = p4type.split("+")
262 base = s[0]
263 mods = ""
264 if len(s) > 1:
265 mods = s[1]
266 return (base, mods)
269 # return the raw p4 type of a file (text, text+ko, etc)
271 def p4_type(file):
272 results = p4CmdList(["fstat", "-T", "headType", file])
273 return results[0]['headType']
276 # Given a type base and modifier, return a regexp matching
277 # the keywords that can be expanded in the file
279 def p4_keywords_regexp_for_type(base, type_mods):
280 if base in ("text", "unicode", "binary"):
281 kwords = None
282 if "ko" in type_mods:
283 kwords = 'Id|Header'
284 elif "k" in type_mods:
285 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
286 else:
287 return None
288 pattern = r"""
289 \$ # Starts with a dollar, followed by...
290 (%s) # one of the keywords, followed by...
291 (:[^$\n]+)? # possibly an old expansion, followed by...
292 \$ # another dollar
293 """ % kwords
294 return pattern
295 else:
296 return None
299 # Given a file, return a regexp matching the possible
300 # RCS keywords that will be expanded, or None for files
301 # with kw expansion turned off.
303 def p4_keywords_regexp_for_file(file):
304 if not os.path.exists(file):
305 return None
306 else:
307 (type_base, type_mods) = split_p4_type(p4_type(file))
308 return p4_keywords_regexp_for_type(type_base, type_mods)
310 def setP4ExecBit(file, mode):
311 # Reopens an already open file and changes the execute bit to match
312 # the execute bit setting in the passed in mode.
314 p4Type = "+x"
316 if not isModeExec(mode):
317 p4Type = getP4OpenedType(file)
318 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
319 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
320 if p4Type[-1] == "+":
321 p4Type = p4Type[0:-1]
323 p4_reopen(p4Type, file)
325 def getP4OpenedType(file):
326 # Returns the perforce file type for the given file.
328 result = p4_read_pipe(["opened", wildcard_encode(file)])
329 match = re.match(".*\((.+)\)\r?$", result)
330 if match:
331 return match.group(1)
332 else:
333 die("Could not determine file type for %s (result: '%s')" % (file, result))
335 # Return the set of all p4 labels
336 def getP4Labels(depotPaths):
337 labels = set()
338 if isinstance(depotPaths,basestring):
339 depotPaths = [depotPaths]
341 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
342 label = l['label']
343 labels.add(label)
345 return labels
347 # Return the set of all git tags
348 def getGitTags():
349 gitTags = set()
350 for line in read_pipe_lines(["git", "tag"]):
351 tag = line.strip()
352 gitTags.add(tag)
353 return gitTags
355 def diffTreePattern():
356 # This is a simple generator for the diff tree regex pattern. This could be
357 # a class variable if this and parseDiffTreeEntry were a part of a class.
358 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
359 while True:
360 yield pattern
362 def parseDiffTreeEntry(entry):
363 """Parses a single diff tree entry into its component elements.
365 See git-diff-tree(1) manpage for details about the format of the diff
366 output. This method returns a dictionary with the following elements:
368 src_mode - The mode of the source file
369 dst_mode - The mode of the destination file
370 src_sha1 - The sha1 for the source file
371 dst_sha1 - The sha1 fr the destination file
372 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
373 status_score - The score for the status (applicable for 'C' and 'R'
374 statuses). This is None if there is no score.
375 src - The path for the source file.
376 dst - The path for the destination file. This is only present for
377 copy or renames. If it is not present, this is None.
379 If the pattern is not matched, None is returned."""
381 match = diffTreePattern().next().match(entry)
382 if match:
383 return {
384 'src_mode': match.group(1),
385 'dst_mode': match.group(2),
386 'src_sha1': match.group(3),
387 'dst_sha1': match.group(4),
388 'status': match.group(5),
389 'status_score': match.group(6),
390 'src': match.group(7),
391 'dst': match.group(10)
393 return None
395 def isModeExec(mode):
396 # Returns True if the given git mode represents an executable file,
397 # otherwise False.
398 return mode[-3:] == "755"
400 def isModeExecChanged(src_mode, dst_mode):
401 return isModeExec(src_mode) != isModeExec(dst_mode)
403 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
405 if isinstance(cmd,basestring):
406 cmd = "-G " + cmd
407 expand = True
408 else:
409 cmd = ["-G"] + cmd
410 expand = False
412 cmd = p4_build_cmd(cmd)
413 if verbose:
414 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
416 # Use a temporary file to avoid deadlocks without
417 # subprocess.communicate(), which would put another copy
418 # of stdout into memory.
419 stdin_file = None
420 if stdin is not None:
421 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
422 if isinstance(stdin,basestring):
423 stdin_file.write(stdin)
424 else:
425 for i in stdin:
426 stdin_file.write(i + '\n')
427 stdin_file.flush()
428 stdin_file.seek(0)
430 p4 = subprocess.Popen(cmd,
431 shell=expand,
432 stdin=stdin_file,
433 stdout=subprocess.PIPE)
435 result = []
436 try:
437 while True:
438 entry = marshal.load(p4.stdout)
439 if cb is not None:
440 cb(entry)
441 else:
442 result.append(entry)
443 except EOFError:
444 pass
445 exitCode = p4.wait()
446 if exitCode != 0:
447 entry = {}
448 entry["p4ExitCode"] = exitCode
449 result.append(entry)
451 return result
453 def p4Cmd(cmd):
454 list = p4CmdList(cmd)
455 result = {}
456 for entry in list:
457 result.update(entry)
458 return result;
460 def p4Where(depotPath):
461 if not depotPath.endswith("/"):
462 depotPath += "/"
463 depotPath = depotPath + "..."
464 outputList = p4CmdList(["where", depotPath])
465 output = None
466 for entry in outputList:
467 if "depotFile" in entry:
468 if entry["depotFile"] == depotPath:
469 output = entry
470 break
471 elif "data" in entry:
472 data = entry.get("data")
473 space = data.find(" ")
474 if data[:space] == depotPath:
475 output = entry
476 break
477 if output == None:
478 return ""
479 if output["code"] == "error":
480 return ""
481 clientPath = ""
482 if "path" in output:
483 clientPath = output.get("path")
484 elif "data" in output:
485 data = output.get("data")
486 lastSpace = data.rfind(" ")
487 clientPath = data[lastSpace + 1:]
489 if clientPath.endswith("..."):
490 clientPath = clientPath[:-3]
491 return clientPath
493 def currentGitBranch():
494 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
496 def isValidGitDir(path):
497 if (os.path.exists(path + "/HEAD")
498 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
499 return True;
500 return False
502 def parseRevision(ref):
503 return read_pipe("git rev-parse %s" % ref).strip()
505 def branchExists(ref):
506 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
507 ignore_error=True)
508 return len(rev) > 0
510 def extractLogMessageFromGitCommit(commit):
511 logMessage = ""
513 ## fixme: title is first line of commit, not 1st paragraph.
514 foundTitle = False
515 for log in read_pipe_lines("git cat-file commit %s" % commit):
516 if not foundTitle:
517 if len(log) == 1:
518 foundTitle = True
519 continue
521 logMessage += log
522 return logMessage
524 def extractSettingsGitLog(log):
525 values = {}
526 for line in log.split("\n"):
527 line = line.strip()
528 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
529 if not m:
530 continue
532 assignments = m.group(1).split (':')
533 for a in assignments:
534 vals = a.split ('=')
535 key = vals[0].strip()
536 val = ('='.join (vals[1:])).strip()
537 if val.endswith ('\"') and val.startswith('"'):
538 val = val[1:-1]
540 values[key] = val
542 paths = values.get("depot-paths")
543 if not paths:
544 paths = values.get("depot-path")
545 if paths:
546 values['depot-paths'] = paths.split(',')
547 return values
549 def gitBranchExists(branch):
550 proc = subprocess.Popen(["git", "rev-parse", branch],
551 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
552 return proc.wait() == 0;
554 _gitConfig = {}
555 def gitConfig(key, args = None): # set args to "--bool", for instance
556 if not _gitConfig.has_key(key):
557 argsFilter = ""
558 if args != None:
559 argsFilter = "%s " % args
560 cmd = "git config %s%s" % (argsFilter, key)
561 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
562 return _gitConfig[key]
564 def gitConfigList(key):
565 if not _gitConfig.has_key(key):
566 _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
567 return _gitConfig[key]
569 def p4BranchesInGit(branchesAreInRemotes = True):
570 branches = {}
572 cmdline = "git rev-parse --symbolic "
573 if branchesAreInRemotes:
574 cmdline += " --remotes"
575 else:
576 cmdline += " --branches"
578 for line in read_pipe_lines(cmdline):
579 line = line.strip()
581 ## only import to p4/
582 if not line.startswith('p4/') or line == "p4/HEAD":
583 continue
584 branch = line
586 # strip off p4
587 branch = re.sub ("^p4/", "", line)
589 branches[branch] = parseRevision(line)
590 return branches
592 def findUpstreamBranchPoint(head = "HEAD"):
593 branches = p4BranchesInGit()
594 # map from depot-path to branch name
595 branchByDepotPath = {}
596 for branch in branches.keys():
597 tip = branches[branch]
598 log = extractLogMessageFromGitCommit(tip)
599 settings = extractSettingsGitLog(log)
600 if settings.has_key("depot-paths"):
601 paths = ",".join(settings["depot-paths"])
602 branchByDepotPath[paths] = "remotes/p4/" + branch
604 settings = None
605 parent = 0
606 while parent < 65535:
607 commit = head + "~%s" % parent
608 log = extractLogMessageFromGitCommit(commit)
609 settings = extractSettingsGitLog(log)
610 if settings.has_key("depot-paths"):
611 paths = ",".join(settings["depot-paths"])
612 if branchByDepotPath.has_key(paths):
613 return [branchByDepotPath[paths], settings]
615 parent = parent + 1
617 return ["", settings]
619 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
620 if not silent:
621 print ("Creating/updating branch(es) in %s based on origin branch(es)"
622 % localRefPrefix)
624 originPrefix = "origin/p4/"
626 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
627 line = line.strip()
628 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
629 continue
631 headName = line[len(originPrefix):]
632 remoteHead = localRefPrefix + headName
633 originHead = line
635 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
636 if (not original.has_key('depot-paths')
637 or not original.has_key('change')):
638 continue
640 update = False
641 if not gitBranchExists(remoteHead):
642 if verbose:
643 print "creating %s" % remoteHead
644 update = True
645 else:
646 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
647 if settings.has_key('change') > 0:
648 if settings['depot-paths'] == original['depot-paths']:
649 originP4Change = int(original['change'])
650 p4Change = int(settings['change'])
651 if originP4Change > p4Change:
652 print ("%s (%s) is newer than %s (%s). "
653 "Updating p4 branch from origin."
654 % (originHead, originP4Change,
655 remoteHead, p4Change))
656 update = True
657 else:
658 print ("Ignoring: %s was imported from %s while "
659 "%s was imported from %s"
660 % (originHead, ','.join(original['depot-paths']),
661 remoteHead, ','.join(settings['depot-paths'])))
663 if update:
664 system("git update-ref %s %s" % (remoteHead, originHead))
666 def originP4BranchesExist():
667 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
669 def p4ChangesForPaths(depotPaths, changeRange):
670 assert depotPaths
671 cmd = ['changes']
672 for p in depotPaths:
673 cmd += ["%s...%s" % (p, changeRange)]
674 output = p4_read_pipe_lines(cmd)
676 changes = {}
677 for line in output:
678 changeNum = int(line.split(" ")[1])
679 changes[changeNum] = True
681 changelist = changes.keys()
682 changelist.sort()
683 return changelist
685 def p4PathStartsWith(path, prefix):
686 # This method tries to remedy a potential mixed-case issue:
688 # If UserA adds //depot/DirA/file1
689 # and UserB adds //depot/dira/file2
691 # we may or may not have a problem. If you have core.ignorecase=true,
692 # we treat DirA and dira as the same directory
693 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
694 if ignorecase:
695 return path.lower().startswith(prefix.lower())
696 return path.startswith(prefix)
698 def getClientSpec():
699 """Look at the p4 client spec, create a View() object that contains
700 all the mappings, and return it."""
702 specList = p4CmdList("client -o")
703 if len(specList) != 1:
704 die('Output from "client -o" is %d lines, expecting 1' %
705 len(specList))
707 # dictionary of all client parameters
708 entry = specList[0]
710 # just the keys that start with "View"
711 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
713 # hold this new View
714 view = View()
716 # append the lines, in order, to the view
717 for view_num in range(len(view_keys)):
718 k = "View%d" % view_num
719 if k not in view_keys:
720 die("Expected view key %s missing" % k)
721 view.append(entry[k])
723 return view
725 def getClientRoot():
726 """Grab the client directory."""
728 output = p4CmdList("client -o")
729 if len(output) != 1:
730 die('Output from "client -o" is %d lines, expecting 1' % len(output))
732 entry = output[0]
733 if "Root" not in entry:
734 die('Client has no "Root"')
736 return entry["Root"]
739 # P4 wildcards are not allowed in filenames. P4 complains
740 # if you simply add them, but you can force it with "-f", in
741 # which case it translates them into %xx encoding internally.
743 def wildcard_decode(path):
744 # Search for and fix just these four characters. Do % last so
745 # that fixing it does not inadvertently create new %-escapes.
746 # Cannot have * in a filename in windows; untested as to
747 # what p4 would do in such a case.
748 if not platform.system() == "Windows":
749 path = path.replace("%2A", "*")
750 path = path.replace("%23", "#") \
751 .replace("%40", "@") \
752 .replace("%25", "%")
753 return path
755 def wildcard_encode(path):
756 # do % first to avoid double-encoding the %s introduced here
757 path = path.replace("%", "%25") \
758 .replace("*", "%2A") \
759 .replace("#", "%23") \
760 .replace("@", "%40")
761 return path
763 def wildcard_present(path):
764 m = re.search("[*#@%]", path)
765 return m is not None
767 class Command:
768 def __init__(self):
769 self.usage = "usage: %prog [options]"
770 self.needsGit = True
771 self.verbose = False
773 class P4UserMap:
774 def __init__(self):
775 self.userMapFromPerforceServer = False
776 self.myP4UserId = None
778 def p4UserId(self):
779 if self.myP4UserId:
780 return self.myP4UserId
782 results = p4CmdList("user -o")
783 for r in results:
784 if r.has_key('User'):
785 self.myP4UserId = r['User']
786 return r['User']
787 die("Could not find your p4 user id")
789 def p4UserIsMe(self, p4User):
790 # return True if the given p4 user is actually me
791 me = self.p4UserId()
792 if not p4User or p4User != me:
793 return False
794 else:
795 return True
797 def getUserCacheFilename(self):
798 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
799 return home + "/.gitp4-usercache.txt"
801 def getUserMapFromPerforceServer(self):
802 if self.userMapFromPerforceServer:
803 return
804 self.users = {}
805 self.emails = {}
807 for output in p4CmdList("users"):
808 if not output.has_key("User"):
809 continue
810 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
811 self.emails[output["Email"]] = output["User"]
814 s = ''
815 for (key, val) in self.users.items():
816 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
818 open(self.getUserCacheFilename(), "wb").write(s)
819 self.userMapFromPerforceServer = True
821 def loadUserMapFromCache(self):
822 self.users = {}
823 self.userMapFromPerforceServer = False
824 try:
825 cache = open(self.getUserCacheFilename(), "rb")
826 lines = cache.readlines()
827 cache.close()
828 for line in lines:
829 entry = line.strip().split("\t")
830 self.users[entry[0]] = entry[1]
831 except IOError:
832 self.getUserMapFromPerforceServer()
834 class P4Debug(Command):
835 def __init__(self):
836 Command.__init__(self)
837 self.options = []
838 self.description = "A tool to debug the output of p4 -G."
839 self.needsGit = False
841 def run(self, args):
842 j = 0
843 for output in p4CmdList(args):
844 print 'Element: %d' % j
845 j += 1
846 print output
847 return True
849 class P4RollBack(Command):
850 def __init__(self):
851 Command.__init__(self)
852 self.options = [
853 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
855 self.description = "A tool to debug the multi-branch import. Don't use :)"
856 self.rollbackLocalBranches = False
858 def run(self, args):
859 if len(args) != 1:
860 return False
861 maxChange = int(args[0])
863 if "p4ExitCode" in p4Cmd("changes -m 1"):
864 die("Problems executing p4");
866 if self.rollbackLocalBranches:
867 refPrefix = "refs/heads/"
868 lines = read_pipe_lines("git rev-parse --symbolic --branches")
869 else:
870 refPrefix = "refs/remotes/"
871 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
873 for line in lines:
874 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
875 line = line.strip()
876 ref = refPrefix + line
877 log = extractLogMessageFromGitCommit(ref)
878 settings = extractSettingsGitLog(log)
880 depotPaths = settings['depot-paths']
881 change = settings['change']
883 changed = False
885 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
886 for p in depotPaths]))) == 0:
887 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
888 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
889 continue
891 while change and int(change) > maxChange:
892 changed = True
893 if self.verbose:
894 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
895 system("git update-ref %s \"%s^\"" % (ref, ref))
896 log = extractLogMessageFromGitCommit(ref)
897 settings = extractSettingsGitLog(log)
900 depotPaths = settings['depot-paths']
901 change = settings['change']
903 if changed:
904 print "%s rewound to %s" % (ref, change)
906 return True
908 class P4Submit(Command, P4UserMap):
910 conflict_behavior_choices = ("ask", "skip", "quit")
912 def __init__(self):
913 Command.__init__(self)
914 P4UserMap.__init__(self)
915 self.options = [
916 optparse.make_option("--origin", dest="origin"),
917 optparse.make_option("-M", dest="detectRenames", action="store_true"),
918 # preserve the user, requires relevant p4 permissions
919 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
920 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
921 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
922 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
923 optparse.make_option("--conflict", dest="conflict_behavior",
924 choices=self.conflict_behavior_choices)
926 self.description = "Submit changes from git to the perforce depot."
927 self.usage += " [name of git branch to submit into perforce depot]"
928 self.origin = ""
929 self.detectRenames = False
930 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
931 self.dry_run = False
932 self.prepare_p4_only = False
933 self.conflict_behavior = None
934 self.isWindows = (platform.system() == "Windows")
935 self.exportLabels = False
936 self.p4HasMoveCommand = p4_has_move_command()
938 def check(self):
939 if len(p4CmdList("opened ...")) > 0:
940 die("You have files opened with perforce! Close them before starting the sync.")
942 def separate_jobs_from_description(self, message):
943 """Extract and return a possible Jobs field in the commit
944 message. It goes into a separate section in the p4 change
945 specification.
947 A jobs line starts with "Jobs:" and looks like a new field
948 in a form. Values are white-space separated on the same
949 line or on following lines that start with a tab.
951 This does not parse and extract the full git commit message
952 like a p4 form. It just sees the Jobs: line as a marker
953 to pass everything from then on directly into the p4 form,
954 but outside the description section.
956 Return a tuple (stripped log message, jobs string)."""
958 m = re.search(r'^Jobs:', message, re.MULTILINE)
959 if m is None:
960 return (message, None)
962 jobtext = message[m.start():]
963 stripped_message = message[:m.start()].rstrip()
964 return (stripped_message, jobtext)
966 def prepareLogMessage(self, template, message, jobs):
967 """Edits the template returned from "p4 change -o" to insert
968 the message in the Description field, and the jobs text in
969 the Jobs field."""
970 result = ""
972 inDescriptionSection = False
974 for line in template.split("\n"):
975 if line.startswith("#"):
976 result += line + "\n"
977 continue
979 if inDescriptionSection:
980 if line.startswith("Files:") or line.startswith("Jobs:"):
981 inDescriptionSection = False
982 # insert Jobs section
983 if jobs:
984 result += jobs + "\n"
985 else:
986 continue
987 else:
988 if line.startswith("Description:"):
989 inDescriptionSection = True
990 line += "\n"
991 for messageLine in message.split("\n"):
992 line += "\t" + messageLine + "\n"
994 result += line + "\n"
996 return result
998 def patchRCSKeywords(self, file, pattern):
999 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1000 (handle, outFileName) = tempfile.mkstemp(dir='.')
1001 try:
1002 outFile = os.fdopen(handle, "w+")
1003 inFile = open(file, "r")
1004 regexp = re.compile(pattern, re.VERBOSE)
1005 for line in inFile.readlines():
1006 line = regexp.sub(r'$\1$', line)
1007 outFile.write(line)
1008 inFile.close()
1009 outFile.close()
1010 # Forcibly overwrite the original file
1011 os.unlink(file)
1012 shutil.move(outFileName, file)
1013 except:
1014 # cleanup our temporary file
1015 os.unlink(outFileName)
1016 print "Failed to strip RCS keywords in %s" % file
1017 raise
1019 print "Patched up RCS keywords in %s" % file
1021 def p4UserForCommit(self,id):
1022 # Return the tuple (perforce user,git email) for a given git commit id
1023 self.getUserMapFromPerforceServer()
1024 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
1025 gitEmail = gitEmail.strip()
1026 if not self.emails.has_key(gitEmail):
1027 return (None,gitEmail)
1028 else:
1029 return (self.emails[gitEmail],gitEmail)
1031 def checkValidP4Users(self,commits):
1032 # check if any git authors cannot be mapped to p4 users
1033 for id in commits:
1034 (user,email) = self.p4UserForCommit(id)
1035 if not user:
1036 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1037 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
1038 print "%s" % msg
1039 else:
1040 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1042 def lastP4Changelist(self):
1043 # Get back the last changelist number submitted in this client spec. This
1044 # then gets used to patch up the username in the change. If the same
1045 # client spec is being used by multiple processes then this might go
1046 # wrong.
1047 results = p4CmdList("client -o") # find the current client
1048 client = None
1049 for r in results:
1050 if r.has_key('Client'):
1051 client = r['Client']
1052 break
1053 if not client:
1054 die("could not get client spec")
1055 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1056 for r in results:
1057 if r.has_key('change'):
1058 return r['change']
1059 die("Could not get changelist number for last submit - cannot patch up user details")
1061 def modifyChangelistUser(self, changelist, newUser):
1062 # fixup the user field of a changelist after it has been submitted.
1063 changes = p4CmdList("change -o %s" % changelist)
1064 if len(changes) != 1:
1065 die("Bad output from p4 change modifying %s to user %s" %
1066 (changelist, newUser))
1068 c = changes[0]
1069 if c['User'] == newUser: return # nothing to do
1070 c['User'] = newUser
1071 input = marshal.dumps(c)
1073 result = p4CmdList("change -f -i", stdin=input)
1074 for r in result:
1075 if r.has_key('code'):
1076 if r['code'] == 'error':
1077 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1078 if r.has_key('data'):
1079 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1080 return
1081 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1083 def canChangeChangelists(self):
1084 # check to see if we have p4 admin or super-user permissions, either of
1085 # which are required to modify changelists.
1086 results = p4CmdList(["protects", self.depotPath])
1087 for r in results:
1088 if r.has_key('perm'):
1089 if r['perm'] == 'admin':
1090 return 1
1091 if r['perm'] == 'super':
1092 return 1
1093 return 0
1095 def prepareSubmitTemplate(self):
1096 """Run "p4 change -o" to grab a change specification template.
1097 This does not use "p4 -G", as it is nice to keep the submission
1098 template in original order, since a human might edit it.
1100 Remove lines in the Files section that show changes to files
1101 outside the depot path we're committing into."""
1103 template = ""
1104 inFilesSection = False
1105 for line in p4_read_pipe_lines(['change', '-o']):
1106 if line.endswith("\r\n"):
1107 line = line[:-2] + "\n"
1108 if inFilesSection:
1109 if line.startswith("\t"):
1110 # path starts and ends with a tab
1111 path = line[1:]
1112 lastTab = path.rfind("\t")
1113 if lastTab != -1:
1114 path = path[:lastTab]
1115 if not p4PathStartsWith(path, self.depotPath):
1116 continue
1117 else:
1118 inFilesSection = False
1119 else:
1120 if line.startswith("Files:"):
1121 inFilesSection = True
1123 template += line
1125 return template
1127 def edit_template(self, template_file):
1128 """Invoke the editor to let the user change the submission
1129 message. Return true if okay to continue with the submit."""
1131 # if configured to skip the editing part, just submit
1132 if gitConfig("git-p4.skipSubmitEdit") == "true":
1133 return True
1135 # look at the modification time, to check later if the user saved
1136 # the file
1137 mtime = os.stat(template_file).st_mtime
1139 # invoke the editor
1140 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1141 editor = os.environ.get("P4EDITOR")
1142 else:
1143 editor = read_pipe("git var GIT_EDITOR").strip()
1144 system(editor + " " + template_file)
1146 # If the file was not saved, prompt to see if this patch should
1147 # be skipped. But skip this verification step if configured so.
1148 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1149 return True
1151 # modification time updated means user saved the file
1152 if os.stat(template_file).st_mtime > mtime:
1153 return True
1155 while True:
1156 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1157 if response == 'y':
1158 return True
1159 if response == 'n':
1160 return False
1162 def applyCommit(self, id):
1163 """Apply one commit, return True if it succeeded."""
1165 print "Applying", read_pipe(["git", "show", "-s",
1166 "--format=format:%h %s", id])
1168 (p4User, gitEmail) = self.p4UserForCommit(id)
1170 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1171 filesToAdd = set()
1172 filesToDelete = set()
1173 editedFiles = set()
1174 pureRenameCopy = set()
1175 filesToChangeExecBit = {}
1177 for line in diff:
1178 diff = parseDiffTreeEntry(line)
1179 modifier = diff['status']
1180 path = diff['src']
1181 if modifier == "M":
1182 p4_edit(path)
1183 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1184 filesToChangeExecBit[path] = diff['dst_mode']
1185 editedFiles.add(path)
1186 elif modifier == "A":
1187 filesToAdd.add(path)
1188 filesToChangeExecBit[path] = diff['dst_mode']
1189 if path in filesToDelete:
1190 filesToDelete.remove(path)
1191 elif modifier == "D":
1192 filesToDelete.add(path)
1193 if path in filesToAdd:
1194 filesToAdd.remove(path)
1195 elif modifier == "C":
1196 src, dest = diff['src'], diff['dst']
1197 p4_integrate(src, dest)
1198 pureRenameCopy.add(dest)
1199 if diff['src_sha1'] != diff['dst_sha1']:
1200 p4_edit(dest)
1201 pureRenameCopy.discard(dest)
1202 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1203 p4_edit(dest)
1204 pureRenameCopy.discard(dest)
1205 filesToChangeExecBit[dest] = diff['dst_mode']
1206 os.unlink(dest)
1207 editedFiles.add(dest)
1208 elif modifier == "R":
1209 src, dest = diff['src'], diff['dst']
1210 if self.p4HasMoveCommand:
1211 p4_edit(src) # src must be open before move
1212 p4_move(src, dest) # opens for (move/delete, move/add)
1213 else:
1214 p4_integrate(src, dest)
1215 if diff['src_sha1'] != diff['dst_sha1']:
1216 p4_edit(dest)
1217 else:
1218 pureRenameCopy.add(dest)
1219 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1220 if not self.p4HasMoveCommand:
1221 p4_edit(dest) # with move: already open, writable
1222 filesToChangeExecBit[dest] = diff['dst_mode']
1223 if not self.p4HasMoveCommand:
1224 os.unlink(dest)
1225 filesToDelete.add(src)
1226 editedFiles.add(dest)
1227 else:
1228 die("unknown modifier %s for %s" % (modifier, path))
1230 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1231 patchcmd = diffcmd + " | git apply "
1232 tryPatchCmd = patchcmd + "--check -"
1233 applyPatchCmd = patchcmd + "--check --apply -"
1234 patch_succeeded = True
1236 if os.system(tryPatchCmd) != 0:
1237 fixed_rcs_keywords = False
1238 patch_succeeded = False
1239 print "Unfortunately applying the change failed!"
1241 # Patch failed, maybe it's just RCS keyword woes. Look through
1242 # the patch to see if that's possible.
1243 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1244 file = None
1245 pattern = None
1246 kwfiles = {}
1247 for file in editedFiles | filesToDelete:
1248 # did this file's delta contain RCS keywords?
1249 pattern = p4_keywords_regexp_for_file(file)
1251 if pattern:
1252 # this file is a possibility...look for RCS keywords.
1253 regexp = re.compile(pattern, re.VERBOSE)
1254 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1255 if regexp.search(line):
1256 if verbose:
1257 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1258 kwfiles[file] = pattern
1259 break
1261 for file in kwfiles:
1262 if verbose:
1263 print "zapping %s with %s" % (line,pattern)
1264 self.patchRCSKeywords(file, kwfiles[file])
1265 fixed_rcs_keywords = True
1267 if fixed_rcs_keywords:
1268 print "Retrying the patch with RCS keywords cleaned up"
1269 if os.system(tryPatchCmd) == 0:
1270 patch_succeeded = True
1272 if not patch_succeeded:
1273 for f in editedFiles:
1274 p4_revert(f)
1275 return False
1278 # Apply the patch for real, and do add/delete/+x handling.
1280 system(applyPatchCmd)
1282 for f in filesToAdd:
1283 p4_add(f)
1284 for f in filesToDelete:
1285 p4_revert(f)
1286 p4_delete(f)
1288 # Set/clear executable bits
1289 for f in filesToChangeExecBit.keys():
1290 mode = filesToChangeExecBit[f]
1291 setP4ExecBit(f, mode)
1294 # Build p4 change description, starting with the contents
1295 # of the git commit message.
1297 logMessage = extractLogMessageFromGitCommit(id)
1298 logMessage = logMessage.strip()
1299 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1301 template = self.prepareSubmitTemplate()
1302 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1304 if self.preserveUser:
1305 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1307 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1308 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1309 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1310 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1312 separatorLine = "######## everything below this line is just the diff #######\n"
1314 # diff
1315 if os.environ.has_key("P4DIFF"):
1316 del(os.environ["P4DIFF"])
1317 diff = ""
1318 for editedFile in editedFiles:
1319 diff += p4_read_pipe(['diff', '-du',
1320 wildcard_encode(editedFile)])
1322 # new file diff
1323 newdiff = ""
1324 for newFile in filesToAdd:
1325 newdiff += "==== new file ====\n"
1326 newdiff += "--- /dev/null\n"
1327 newdiff += "+++ %s\n" % newFile
1328 f = open(newFile, "r")
1329 for line in f.readlines():
1330 newdiff += "+" + line
1331 f.close()
1333 # change description file: submitTemplate, separatorLine, diff, newdiff
1334 (handle, fileName) = tempfile.mkstemp()
1335 tmpFile = os.fdopen(handle, "w+")
1336 if self.isWindows:
1337 submitTemplate = submitTemplate.replace("\n", "\r\n")
1338 separatorLine = separatorLine.replace("\n", "\r\n")
1339 newdiff = newdiff.replace("\n", "\r\n")
1340 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1341 tmpFile.close()
1343 if self.prepare_p4_only:
1345 # Leave the p4 tree prepared, and the submit template around
1346 # and let the user decide what to do next
1348 print
1349 print "P4 workspace prepared for submission."
1350 print "To submit or revert, go to client workspace"
1351 print " " + self.clientPath
1352 print
1353 print "To submit, use \"p4 submit\" to write a new description,"
1354 print "or \"p4 submit -i %s\" to use the one prepared by" \
1355 " \"git p4\"." % fileName
1356 print "You can delete the file \"%s\" when finished." % fileName
1358 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1359 print "To preserve change ownership by user %s, you must\n" \
1360 "do \"p4 change -f <change>\" after submitting and\n" \
1361 "edit the User field."
1362 if pureRenameCopy:
1363 print "After submitting, renamed files must be re-synced."
1364 print "Invoke \"p4 sync -f\" on each of these files:"
1365 for f in pureRenameCopy:
1366 print " " + f
1368 print
1369 print "To revert the changes, use \"p4 revert ...\", and delete"
1370 print "the submit template file \"%s\"" % fileName
1371 if filesToAdd:
1372 print "Since the commit adds new files, they must be deleted:"
1373 for f in filesToAdd:
1374 print " " + f
1375 print
1376 return True
1379 # Let the user edit the change description, then submit it.
1381 if self.edit_template(fileName):
1382 # read the edited message and submit
1383 ret = True
1384 tmpFile = open(fileName, "rb")
1385 message = tmpFile.read()
1386 tmpFile.close()
1387 submitTemplate = message[:message.index(separatorLine)]
1388 if self.isWindows:
1389 submitTemplate = submitTemplate.replace("\r\n", "\n")
1390 p4_write_pipe(['submit', '-i'], submitTemplate)
1392 if self.preserveUser:
1393 if p4User:
1394 # Get last changelist number. Cannot easily get it from
1395 # the submit command output as the output is
1396 # unmarshalled.
1397 changelist = self.lastP4Changelist()
1398 self.modifyChangelistUser(changelist, p4User)
1400 # The rename/copy happened by applying a patch that created a
1401 # new file. This leaves it writable, which confuses p4.
1402 for f in pureRenameCopy:
1403 p4_sync(f, "-f")
1405 else:
1406 # skip this patch
1407 ret = False
1408 print "Submission cancelled, undoing p4 changes."
1409 for f in editedFiles:
1410 p4_revert(f)
1411 for f in filesToAdd:
1412 p4_revert(f)
1413 os.remove(f)
1414 for f in filesToDelete:
1415 p4_revert(f)
1417 os.remove(fileName)
1418 return ret
1420 # Export git tags as p4 labels. Create a p4 label and then tag
1421 # with that.
1422 def exportGitTags(self, gitTags):
1423 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1424 if len(validLabelRegexp) == 0:
1425 validLabelRegexp = defaultLabelRegexp
1426 m = re.compile(validLabelRegexp)
1428 for name in gitTags:
1430 if not m.match(name):
1431 if verbose:
1432 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1433 continue
1435 # Get the p4 commit this corresponds to
1436 logMessage = extractLogMessageFromGitCommit(name)
1437 values = extractSettingsGitLog(logMessage)
1439 if not values.has_key('change'):
1440 # a tag pointing to something not sent to p4; ignore
1441 if verbose:
1442 print "git tag %s does not give a p4 commit" % name
1443 continue
1444 else:
1445 changelist = values['change']
1447 # Get the tag details.
1448 inHeader = True
1449 isAnnotated = False
1450 body = []
1451 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1452 l = l.strip()
1453 if inHeader:
1454 if re.match(r'tag\s+', l):
1455 isAnnotated = True
1456 elif re.match(r'\s*$', l):
1457 inHeader = False
1458 continue
1459 else:
1460 body.append(l)
1462 if not isAnnotated:
1463 body = ["lightweight tag imported by git p4\n"]
1465 # Create the label - use the same view as the client spec we are using
1466 clientSpec = getClientSpec()
1468 labelTemplate = "Label: %s\n" % name
1469 labelTemplate += "Description:\n"
1470 for b in body:
1471 labelTemplate += "\t" + b + "\n"
1472 labelTemplate += "View:\n"
1473 for mapping in clientSpec.mappings:
1474 labelTemplate += "\t%s\n" % mapping.depot_side.path
1476 if self.dry_run:
1477 print "Would create p4 label %s for tag" % name
1478 elif self.prepare_p4_only:
1479 print "Not creating p4 label %s for tag due to option" \
1480 " --prepare-p4-only" % name
1481 else:
1482 p4_write_pipe(["label", "-i"], labelTemplate)
1484 # Use the label
1485 p4_system(["tag", "-l", name] +
1486 ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1488 if verbose:
1489 print "created p4 label for tag %s" % name
1491 def run(self, args):
1492 if len(args) == 0:
1493 self.master = currentGitBranch()
1494 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1495 die("Detecting current git branch failed!")
1496 elif len(args) == 1:
1497 self.master = args[0]
1498 if not branchExists(self.master):
1499 die("Branch %s does not exist" % self.master)
1500 else:
1501 return False
1503 allowSubmit = gitConfig("git-p4.allowSubmit")
1504 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1505 die("%s is not in git-p4.allowSubmit" % self.master)
1507 [upstream, settings] = findUpstreamBranchPoint()
1508 self.depotPath = settings['depot-paths'][0]
1509 if len(self.origin) == 0:
1510 self.origin = upstream
1512 if self.preserveUser:
1513 if not self.canChangeChangelists():
1514 die("Cannot preserve user names without p4 super-user or admin permissions")
1516 # if not set from the command line, try the config file
1517 if self.conflict_behavior is None:
1518 val = gitConfig("git-p4.conflict")
1519 if val:
1520 if val not in self.conflict_behavior_choices:
1521 die("Invalid value '%s' for config git-p4.conflict" % val)
1522 else:
1523 val = "ask"
1524 self.conflict_behavior = val
1526 if self.verbose:
1527 print "Origin branch is " + self.origin
1529 if len(self.depotPath) == 0:
1530 print "Internal error: cannot locate perforce depot path from existing branches"
1531 sys.exit(128)
1533 self.useClientSpec = False
1534 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1535 self.useClientSpec = True
1536 if self.useClientSpec:
1537 self.clientSpecDirs = getClientSpec()
1539 if self.useClientSpec:
1540 # all files are relative to the client spec
1541 self.clientPath = getClientRoot()
1542 else:
1543 self.clientPath = p4Where(self.depotPath)
1545 if self.clientPath == "":
1546 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1548 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1549 self.oldWorkingDirectory = os.getcwd()
1551 # ensure the clientPath exists
1552 new_client_dir = False
1553 if not os.path.exists(self.clientPath):
1554 new_client_dir = True
1555 os.makedirs(self.clientPath)
1557 chdir(self.clientPath)
1558 if self.dry_run:
1559 print "Would synchronize p4 checkout in %s" % self.clientPath
1560 else:
1561 print "Synchronizing p4 checkout..."
1562 if new_client_dir:
1563 # old one was destroyed, and maybe nobody told p4
1564 p4_sync("...", "-f")
1565 else:
1566 p4_sync("...")
1567 self.check()
1569 commits = []
1570 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1571 commits.append(line.strip())
1572 commits.reverse()
1574 if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1575 self.checkAuthorship = False
1576 else:
1577 self.checkAuthorship = True
1579 if self.preserveUser:
1580 self.checkValidP4Users(commits)
1583 # Build up a set of options to be passed to diff when
1584 # submitting each commit to p4.
1586 if self.detectRenames:
1587 # command-line -M arg
1588 self.diffOpts = "-M"
1589 else:
1590 # If not explicitly set check the config variable
1591 detectRenames = gitConfig("git-p4.detectRenames")
1593 if detectRenames.lower() == "false" or detectRenames == "":
1594 self.diffOpts = ""
1595 elif detectRenames.lower() == "true":
1596 self.diffOpts = "-M"
1597 else:
1598 self.diffOpts = "-M%s" % detectRenames
1600 # no command-line arg for -C or --find-copies-harder, just
1601 # config variables
1602 detectCopies = gitConfig("git-p4.detectCopies")
1603 if detectCopies.lower() == "false" or detectCopies == "":
1604 pass
1605 elif detectCopies.lower() == "true":
1606 self.diffOpts += " -C"
1607 else:
1608 self.diffOpts += " -C%s" % detectCopies
1610 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1611 self.diffOpts += " --find-copies-harder"
1614 # Apply the commits, one at a time. On failure, ask if should
1615 # continue to try the rest of the patches, or quit.
1617 if self.dry_run:
1618 print "Would apply"
1619 applied = []
1620 last = len(commits) - 1
1621 for i, commit in enumerate(commits):
1622 if self.dry_run:
1623 print " ", read_pipe(["git", "show", "-s",
1624 "--format=format:%h %s", commit])
1625 ok = True
1626 else:
1627 ok = self.applyCommit(commit)
1628 if ok:
1629 applied.append(commit)
1630 else:
1631 if self.prepare_p4_only and i < last:
1632 print "Processing only the first commit due to option" \
1633 " --prepare-p4-only"
1634 break
1635 if i < last:
1636 quit = False
1637 while True:
1638 # prompt for what to do, or use the option/variable
1639 if self.conflict_behavior == "ask":
1640 print "What do you want to do?"
1641 response = raw_input("[s]kip this commit but apply"
1642 " the rest, or [q]uit? ")
1643 if not response:
1644 continue
1645 elif self.conflict_behavior == "skip":
1646 response = "s"
1647 elif self.conflict_behavior == "quit":
1648 response = "q"
1649 else:
1650 die("Unknown conflict_behavior '%s'" %
1651 self.conflict_behavior)
1653 if response[0] == "s":
1654 print "Skipping this commit, but applying the rest"
1655 break
1656 if response[0] == "q":
1657 print "Quitting"
1658 quit = True
1659 break
1660 if quit:
1661 break
1663 chdir(self.oldWorkingDirectory)
1665 if self.dry_run:
1666 pass
1667 elif self.prepare_p4_only:
1668 pass
1669 elif len(commits) == len(applied):
1670 print "All commits applied!"
1672 sync = P4Sync()
1673 sync.run([])
1675 rebase = P4Rebase()
1676 rebase.rebase()
1678 else:
1679 if len(applied) == 0:
1680 print "No commits applied."
1681 else:
1682 print "Applied only the commits marked with '*':"
1683 for c in commits:
1684 if c in applied:
1685 star = "*"
1686 else:
1687 star = " "
1688 print star, read_pipe(["git", "show", "-s",
1689 "--format=format:%h %s", c])
1690 print "You will have to do 'git p4 sync' and rebase."
1692 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1693 self.exportLabels = True
1695 if self.exportLabels:
1696 p4Labels = getP4Labels(self.depotPath)
1697 gitTags = getGitTags()
1699 missingGitTags = gitTags - p4Labels
1700 self.exportGitTags(missingGitTags)
1702 # exit with error unless everything applied perfecly
1703 if len(commits) != len(applied):
1704 sys.exit(1)
1706 return True
1708 class View(object):
1709 """Represent a p4 view ("p4 help views"), and map files in a
1710 repo according to the view."""
1712 class Path(object):
1713 """A depot or client path, possibly containing wildcards.
1714 The only one supported is ... at the end, currently.
1715 Initialize with the full path, with //depot or //client."""
1717 def __init__(self, path, is_depot):
1718 self.path = path
1719 self.is_depot = is_depot
1720 self.find_wildcards()
1721 # remember the prefix bit, useful for relative mappings
1722 m = re.match("(//[^/]+/)", self.path)
1723 if not m:
1724 die("Path %s does not start with //prefix/" % self.path)
1725 prefix = m.group(1)
1726 if not self.is_depot:
1727 # strip //client/ on client paths
1728 self.path = self.path[len(prefix):]
1730 def find_wildcards(self):
1731 """Make sure wildcards are valid, and set up internal
1732 variables."""
1734 self.ends_triple_dot = False
1735 # There are three wildcards allowed in p4 views
1736 # (see "p4 help views"). This code knows how to
1737 # handle "..." (only at the end), but cannot deal with
1738 # "%%n" or "*". Only check the depot_side, as p4 should
1739 # validate that the client_side matches too.
1740 if re.search(r'%%[1-9]', self.path):
1741 die("Can't handle %%n wildcards in view: %s" % self.path)
1742 if self.path.find("*") >= 0:
1743 die("Can't handle * wildcards in view: %s" % self.path)
1744 triple_dot_index = self.path.find("...")
1745 if triple_dot_index >= 0:
1746 if triple_dot_index != len(self.path) - 3:
1747 die("Can handle only single ... wildcard, at end: %s" %
1748 self.path)
1749 self.ends_triple_dot = True
1751 def ensure_compatible(self, other_path):
1752 """Make sure the wildcards agree."""
1753 if self.ends_triple_dot != other_path.ends_triple_dot:
1754 die("Both paths must end with ... if either does;\n" +
1755 "paths: %s %s" % (self.path, other_path.path))
1757 def match_wildcards(self, test_path):
1758 """See if this test_path matches us, and fill in the value
1759 of the wildcards if so. Returns a tuple of
1760 (True|False, wildcards[]). For now, only the ... at end
1761 is supported, so at most one wildcard."""
1762 if self.ends_triple_dot:
1763 dotless = self.path[:-3]
1764 if test_path.startswith(dotless):
1765 wildcard = test_path[len(dotless):]
1766 return (True, [ wildcard ])
1767 else:
1768 if test_path == self.path:
1769 return (True, [])
1770 return (False, [])
1772 def match(self, test_path):
1773 """Just return if it matches; don't bother with the wildcards."""
1774 b, _ = self.match_wildcards(test_path)
1775 return b
1777 def fill_in_wildcards(self, wildcards):
1778 """Return the relative path, with the wildcards filled in
1779 if there are any."""
1780 if self.ends_triple_dot:
1781 return self.path[:-3] + wildcards[0]
1782 else:
1783 return self.path
1785 class Mapping(object):
1786 def __init__(self, depot_side, client_side, overlay, exclude):
1787 # depot_side is without the trailing /... if it had one
1788 self.depot_side = View.Path(depot_side, is_depot=True)
1789 self.client_side = View.Path(client_side, is_depot=False)
1790 self.overlay = overlay # started with "+"
1791 self.exclude = exclude # started with "-"
1792 assert not (self.overlay and self.exclude)
1793 self.depot_side.ensure_compatible(self.client_side)
1795 def __str__(self):
1796 c = " "
1797 if self.overlay:
1798 c = "+"
1799 if self.exclude:
1800 c = "-"
1801 return "View.Mapping: %s%s -> %s" % \
1802 (c, self.depot_side.path, self.client_side.path)
1804 def map_depot_to_client(self, depot_path):
1805 """Calculate the client path if using this mapping on the
1806 given depot path; does not consider the effect of other
1807 mappings in a view. Even excluded mappings are returned."""
1808 matches, wildcards = self.depot_side.match_wildcards(depot_path)
1809 if not matches:
1810 return ""
1811 client_path = self.client_side.fill_in_wildcards(wildcards)
1812 return client_path
1815 # View methods
1817 def __init__(self):
1818 self.mappings = []
1820 def append(self, view_line):
1821 """Parse a view line, splitting it into depot and client
1822 sides. Append to self.mappings, preserving order."""
1824 # Split the view line into exactly two words. P4 enforces
1825 # structure on these lines that simplifies this quite a bit.
1827 # Either or both words may be double-quoted.
1828 # Single quotes do not matter.
1829 # Double-quote marks cannot occur inside the words.
1830 # A + or - prefix is also inside the quotes.
1831 # There are no quotes unless they contain a space.
1832 # The line is already white-space stripped.
1833 # The two words are separated by a single space.
1835 if view_line[0] == '"':
1836 # First word is double quoted. Find its end.
1837 close_quote_index = view_line.find('"', 1)
1838 if close_quote_index <= 0:
1839 die("No first-word closing quote found: %s" % view_line)
1840 depot_side = view_line[1:close_quote_index]
1841 # skip closing quote and space
1842 rhs_index = close_quote_index + 1 + 1
1843 else:
1844 space_index = view_line.find(" ")
1845 if space_index <= 0:
1846 die("No word-splitting space found: %s" % view_line)
1847 depot_side = view_line[0:space_index]
1848 rhs_index = space_index + 1
1850 if view_line[rhs_index] == '"':
1851 # Second word is double quoted. Make sure there is a
1852 # double quote at the end too.
1853 if not view_line.endswith('"'):
1854 die("View line with rhs quote should end with one: %s" %
1855 view_line)
1856 # skip the quotes
1857 client_side = view_line[rhs_index+1:-1]
1858 else:
1859 client_side = view_line[rhs_index:]
1861 # prefix + means overlay on previous mapping
1862 overlay = False
1863 if depot_side.startswith("+"):
1864 overlay = True
1865 depot_side = depot_side[1:]
1867 # prefix - means exclude this path
1868 exclude = False
1869 if depot_side.startswith("-"):
1870 exclude = True
1871 depot_side = depot_side[1:]
1873 m = View.Mapping(depot_side, client_side, overlay, exclude)
1874 self.mappings.append(m)
1876 def map_in_client(self, depot_path):
1877 """Return the relative location in the client where this
1878 depot file should live. Returns "" if the file should
1879 not be mapped in the client."""
1881 paths_filled = []
1882 client_path = ""
1884 # look at later entries first
1885 for m in self.mappings[::-1]:
1887 # see where will this path end up in the client
1888 p = m.map_depot_to_client(depot_path)
1890 if p == "":
1891 # Depot path does not belong in client. Must remember
1892 # this, as previous items should not cause files to
1893 # exist in this path either. Remember that the list is
1894 # being walked from the end, which has higher precedence.
1895 # Overlap mappings do not exclude previous mappings.
1896 if not m.overlay:
1897 paths_filled.append(m.client_side)
1899 else:
1900 # This mapping matched; no need to search any further.
1901 # But, the mapping could be rejected if the client path
1902 # has already been claimed by an earlier mapping (i.e.
1903 # one later in the list, which we are walking backwards).
1904 already_mapped_in_client = False
1905 for f in paths_filled:
1906 # this is View.Path.match
1907 if f.match(p):
1908 already_mapped_in_client = True
1909 break
1910 if not already_mapped_in_client:
1911 # Include this file, unless it is from a line that
1912 # explicitly said to exclude it.
1913 if not m.exclude:
1914 client_path = p
1916 # a match, even if rejected, always stops the search
1917 break
1919 return client_path
1921 class P4Sync(Command, P4UserMap):
1922 delete_actions = ( "delete", "move/delete", "purge" )
1924 def __init__(self):
1925 Command.__init__(self)
1926 P4UserMap.__init__(self)
1927 self.options = [
1928 optparse.make_option("--branch", dest="branch"),
1929 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1930 optparse.make_option("--changesfile", dest="changesFile"),
1931 optparse.make_option("--silent", dest="silent", action="store_true"),
1932 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1933 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1934 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1935 help="Import into refs/heads/ , not refs/remotes"),
1936 optparse.make_option("--max-changes", dest="maxChanges"),
1937 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1938 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1939 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1940 help="Only sync files that are included in the Perforce Client Spec")
1942 self.description = """Imports from Perforce into a git repository.\n
1943 example:
1944 //depot/my/project/ -- to import the current head
1945 //depot/my/project/@all -- to import everything
1946 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1948 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1950 self.usage += " //depot/path[@revRange]"
1951 self.silent = False
1952 self.createdBranches = set()
1953 self.committedChanges = set()
1954 self.branch = ""
1955 self.detectBranches = False
1956 self.detectLabels = False
1957 self.importLabels = False
1958 self.changesFile = ""
1959 self.syncWithOrigin = True
1960 self.importIntoRemotes = True
1961 self.maxChanges = ""
1962 self.isWindows = (platform.system() == "Windows")
1963 self.keepRepoPath = False
1964 self.depotPaths = None
1965 self.p4BranchesInGit = []
1966 self.cloneExclude = []
1967 self.useClientSpec = False
1968 self.useClientSpec_from_options = False
1969 self.clientSpecDirs = None
1970 self.tempBranches = []
1971 self.tempBranchLocation = "git-p4-tmp"
1973 if gitConfig("git-p4.syncFromOrigin") == "false":
1974 self.syncWithOrigin = False
1976 # Force a checkpoint in fast-import and wait for it to finish
1977 def checkpoint(self):
1978 self.gitStream.write("checkpoint\n\n")
1979 self.gitStream.write("progress checkpoint\n\n")
1980 out = self.gitOutput.readline()
1981 if self.verbose:
1982 print "checkpoint finished: " + out
1984 def extractFilesFromCommit(self, commit):
1985 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1986 for path in self.cloneExclude]
1987 files = []
1988 fnum = 0
1989 while commit.has_key("depotFile%s" % fnum):
1990 path = commit["depotFile%s" % fnum]
1992 if [p for p in self.cloneExclude
1993 if p4PathStartsWith(path, p)]:
1994 found = False
1995 else:
1996 found = [p for p in self.depotPaths
1997 if p4PathStartsWith(path, p)]
1998 if not found:
1999 fnum = fnum + 1
2000 continue
2002 file = {}
2003 file["path"] = path
2004 file["rev"] = commit["rev%s" % fnum]
2005 file["action"] = commit["action%s" % fnum]
2006 file["type"] = commit["type%s" % fnum]
2007 files.append(file)
2008 fnum = fnum + 1
2009 return files
2011 def stripRepoPath(self, path, prefixes):
2012 """When streaming files, this is called to map a p4 depot path
2013 to where it should go in git. The prefixes are either
2014 self.depotPaths, or self.branchPrefixes in the case of
2015 branch detection."""
2017 if self.useClientSpec:
2018 # branch detection moves files up a level (the branch name)
2019 # from what client spec interpretation gives
2020 path = self.clientSpecDirs.map_in_client(path)
2021 if self.detectBranches:
2022 for b in self.knownBranches:
2023 if path.startswith(b + "/"):
2024 path = path[len(b)+1:]
2026 elif self.keepRepoPath:
2027 # Preserve everything in relative path name except leading
2028 # //depot/; just look at first prefix as they all should
2029 # be in the same depot.
2030 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2031 if p4PathStartsWith(path, depot):
2032 path = path[len(depot):]
2034 else:
2035 for p in prefixes:
2036 if p4PathStartsWith(path, p):
2037 path = path[len(p):]
2038 break
2040 path = wildcard_decode(path)
2041 return path
2043 def splitFilesIntoBranches(self, commit):
2044 """Look at each depotFile in the commit to figure out to what
2045 branch it belongs."""
2047 branches = {}
2048 fnum = 0
2049 while commit.has_key("depotFile%s" % fnum):
2050 path = commit["depotFile%s" % fnum]
2051 found = [p for p in self.depotPaths
2052 if p4PathStartsWith(path, p)]
2053 if not found:
2054 fnum = fnum + 1
2055 continue
2057 file = {}
2058 file["path"] = path
2059 file["rev"] = commit["rev%s" % fnum]
2060 file["action"] = commit["action%s" % fnum]
2061 file["type"] = commit["type%s" % fnum]
2062 fnum = fnum + 1
2064 # start with the full relative path where this file would
2065 # go in a p4 client
2066 if self.useClientSpec:
2067 relPath = self.clientSpecDirs.map_in_client(path)
2068 else:
2069 relPath = self.stripRepoPath(path, self.depotPaths)
2071 for branch in self.knownBranches.keys():
2072 # add a trailing slash so that a commit into qt/4.2foo
2073 # doesn't end up in qt/4.2, e.g.
2074 if relPath.startswith(branch + "/"):
2075 if branch not in branches:
2076 branches[branch] = []
2077 branches[branch].append(file)
2078 break
2080 return branches
2082 # output one file from the P4 stream
2083 # - helper for streamP4Files
2085 def streamOneP4File(self, file, contents):
2086 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2087 if verbose:
2088 sys.stderr.write("%s\n" % relPath)
2090 (type_base, type_mods) = split_p4_type(file["type"])
2092 git_mode = "100644"
2093 if "x" in type_mods:
2094 git_mode = "100755"
2095 if type_base == "symlink":
2096 git_mode = "120000"
2097 # p4 print on a symlink contains "target\n"; remove the newline
2098 data = ''.join(contents)
2099 contents = [data[:-1]]
2101 if type_base == "utf16":
2102 # p4 delivers different text in the python output to -G
2103 # than it does when using "print -o", or normal p4 client
2104 # operations. utf16 is converted to ascii or utf8, perhaps.
2105 # But ascii text saved as -t utf16 is completely mangled.
2106 # Invoke print -o to get the real contents.
2107 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2108 contents = [ text ]
2110 if type_base == "apple":
2111 # Apple filetype files will be streamed as a concatenation of
2112 # its appledouble header and the contents. This is useless
2113 # on both macs and non-macs. If using "print -q -o xx", it
2114 # will create "xx" with the data, and "%xx" with the header.
2115 # This is also not very useful.
2117 # Ideally, someday, this script can learn how to generate
2118 # appledouble files directly and import those to git, but
2119 # non-mac machines can never find a use for apple filetype.
2120 print "\nIgnoring apple filetype file %s" % file['depotFile']
2121 return
2123 # Perhaps windows wants unicode, utf16 newlines translated too;
2124 # but this is not doing it.
2125 if self.isWindows and type_base == "text":
2126 mangled = []
2127 for data in contents:
2128 data = data.replace("\r\n", "\n")
2129 mangled.append(data)
2130 contents = mangled
2132 # Note that we do not try to de-mangle keywords on utf16 files,
2133 # even though in theory somebody may want that.
2134 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2135 if pattern:
2136 regexp = re.compile(pattern, re.VERBOSE)
2137 text = ''.join(contents)
2138 text = regexp.sub(r'$\1$', text)
2139 contents = [ text ]
2141 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2143 # total length...
2144 length = 0
2145 for d in contents:
2146 length = length + len(d)
2148 self.gitStream.write("data %d\n" % length)
2149 for d in contents:
2150 self.gitStream.write(d)
2151 self.gitStream.write("\n")
2153 def streamOneP4Deletion(self, file):
2154 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2155 if verbose:
2156 sys.stderr.write("delete %s\n" % relPath)
2157 self.gitStream.write("D %s\n" % relPath)
2159 # handle another chunk of streaming data
2160 def streamP4FilesCb(self, marshalled):
2162 # catch p4 errors and complain
2163 err = None
2164 if "code" in marshalled:
2165 if marshalled["code"] == "error":
2166 if "data" in marshalled:
2167 err = marshalled["data"].rstrip()
2168 if err:
2169 f = None
2170 if self.stream_have_file_info:
2171 if "depotFile" in self.stream_file:
2172 f = self.stream_file["depotFile"]
2173 # force a failure in fast-import, else an empty
2174 # commit will be made
2175 self.gitStream.write("\n")
2176 self.gitStream.write("die-now\n")
2177 self.gitStream.close()
2178 # ignore errors, but make sure it exits first
2179 self.importProcess.wait()
2180 if f:
2181 die("Error from p4 print for %s: %s" % (f, err))
2182 else:
2183 die("Error from p4 print: %s" % err)
2185 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2186 # start of a new file - output the old one first
2187 self.streamOneP4File(self.stream_file, self.stream_contents)
2188 self.stream_file = {}
2189 self.stream_contents = []
2190 self.stream_have_file_info = False
2192 # pick up the new file information... for the
2193 # 'data' field we need to append to our array
2194 for k in marshalled.keys():
2195 if k == 'data':
2196 self.stream_contents.append(marshalled['data'])
2197 else:
2198 self.stream_file[k] = marshalled[k]
2200 self.stream_have_file_info = True
2202 # Stream directly from "p4 files" into "git fast-import"
2203 def streamP4Files(self, files):
2204 filesForCommit = []
2205 filesToRead = []
2206 filesToDelete = []
2208 for f in files:
2209 # if using a client spec, only add the files that have
2210 # a path in the client
2211 if self.clientSpecDirs:
2212 if self.clientSpecDirs.map_in_client(f['path']) == "":
2213 continue
2215 filesForCommit.append(f)
2216 if f['action'] in self.delete_actions:
2217 filesToDelete.append(f)
2218 else:
2219 filesToRead.append(f)
2221 # deleted files...
2222 for f in filesToDelete:
2223 self.streamOneP4Deletion(f)
2225 if len(filesToRead) > 0:
2226 self.stream_file = {}
2227 self.stream_contents = []
2228 self.stream_have_file_info = False
2230 # curry self argument
2231 def streamP4FilesCbSelf(entry):
2232 self.streamP4FilesCb(entry)
2234 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2236 p4CmdList(["-x", "-", "print"],
2237 stdin=fileArgs,
2238 cb=streamP4FilesCbSelf)
2240 # do the last chunk
2241 if self.stream_file.has_key('depotFile'):
2242 self.streamOneP4File(self.stream_file, self.stream_contents)
2244 def make_email(self, userid):
2245 if userid in self.users:
2246 return self.users[userid]
2247 else:
2248 return "%s <a@b>" % userid
2250 # Stream a p4 tag
2251 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2252 if verbose:
2253 print "writing tag %s for commit %s" % (labelName, commit)
2254 gitStream.write("tag %s\n" % labelName)
2255 gitStream.write("from %s\n" % commit)
2257 if labelDetails.has_key('Owner'):
2258 owner = labelDetails["Owner"]
2259 else:
2260 owner = None
2262 # Try to use the owner of the p4 label, or failing that,
2263 # the current p4 user id.
2264 if owner:
2265 email = self.make_email(owner)
2266 else:
2267 email = self.make_email(self.p4UserId())
2268 tagger = "%s %s %s" % (email, epoch, self.tz)
2270 gitStream.write("tagger %s\n" % tagger)
2272 print "labelDetails=",labelDetails
2273 if labelDetails.has_key('Description'):
2274 description = labelDetails['Description']
2275 else:
2276 description = 'Label from git p4'
2278 gitStream.write("data %d\n" % len(description))
2279 gitStream.write(description)
2280 gitStream.write("\n")
2282 def commit(self, details, files, branch, parent = ""):
2283 epoch = details["time"]
2284 author = details["user"]
2286 if self.verbose:
2287 print "commit into %s" % branch
2289 # start with reading files; if that fails, we should not
2290 # create a commit.
2291 new_files = []
2292 for f in files:
2293 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2294 new_files.append (f)
2295 else:
2296 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2298 self.gitStream.write("commit %s\n" % branch)
2299 # gitStream.write("mark :%s\n" % details["change"])
2300 self.committedChanges.add(int(details["change"]))
2301 committer = ""
2302 if author not in self.users:
2303 self.getUserMapFromPerforceServer()
2304 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2306 self.gitStream.write("committer %s\n" % committer)
2308 self.gitStream.write("data <<EOT\n")
2309 self.gitStream.write(details["desc"])
2310 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2311 (','.join(self.branchPrefixes), details["change"]))
2312 if len(details['options']) > 0:
2313 self.gitStream.write(": options = %s" % details['options'])
2314 self.gitStream.write("]\nEOT\n\n")
2316 if len(parent) > 0:
2317 if self.verbose:
2318 print "parent %s" % parent
2319 self.gitStream.write("from %s\n" % parent)
2321 self.streamP4Files(new_files)
2322 self.gitStream.write("\n")
2324 change = int(details["change"])
2326 if self.labels.has_key(change):
2327 label = self.labels[change]
2328 labelDetails = label[0]
2329 labelRevisions = label[1]
2330 if self.verbose:
2331 print "Change %s is labelled %s" % (change, labelDetails)
2333 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2334 for p in self.branchPrefixes])
2336 if len(files) == len(labelRevisions):
2338 cleanedFiles = {}
2339 for info in files:
2340 if info["action"] in self.delete_actions:
2341 continue
2342 cleanedFiles[info["depotFile"]] = info["rev"]
2344 if cleanedFiles == labelRevisions:
2345 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2347 else:
2348 if not self.silent:
2349 print ("Tag %s does not match with change %s: files do not match."
2350 % (labelDetails["label"], change))
2352 else:
2353 if not self.silent:
2354 print ("Tag %s does not match with change %s: file count is different."
2355 % (labelDetails["label"], change))
2357 # Build a dictionary of changelists and labels, for "detect-labels" option.
2358 def getLabels(self):
2359 self.labels = {}
2361 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2362 if len(l) > 0 and not self.silent:
2363 print "Finding files belonging to labels in %s" % `self.depotPaths`
2365 for output in l:
2366 label = output["label"]
2367 revisions = {}
2368 newestChange = 0
2369 if self.verbose:
2370 print "Querying files for label %s" % label
2371 for file in p4CmdList(["files"] +
2372 ["%s...@%s" % (p, label)
2373 for p in self.depotPaths]):
2374 revisions[file["depotFile"]] = file["rev"]
2375 change = int(file["change"])
2376 if change > newestChange:
2377 newestChange = change
2379 self.labels[newestChange] = [output, revisions]
2381 if self.verbose:
2382 print "Label changes: %s" % self.labels.keys()
2384 # Import p4 labels as git tags. A direct mapping does not
2385 # exist, so assume that if all the files are at the same revision
2386 # then we can use that, or it's something more complicated we should
2387 # just ignore.
2388 def importP4Labels(self, stream, p4Labels):
2389 if verbose:
2390 print "import p4 labels: " + ' '.join(p4Labels)
2392 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2393 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2394 if len(validLabelRegexp) == 0:
2395 validLabelRegexp = defaultLabelRegexp
2396 m = re.compile(validLabelRegexp)
2398 for name in p4Labels:
2399 commitFound = False
2401 if not m.match(name):
2402 if verbose:
2403 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2404 continue
2406 if name in ignoredP4Labels:
2407 continue
2409 labelDetails = p4CmdList(['label', "-o", name])[0]
2411 # get the most recent changelist for each file in this label
2412 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2413 for p in self.depotPaths])
2415 if change.has_key('change'):
2416 # find the corresponding git commit; take the oldest commit
2417 changelist = int(change['change'])
2418 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2419 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2420 if len(gitCommit) == 0:
2421 print "could not find git commit for changelist %d" % changelist
2422 else:
2423 gitCommit = gitCommit.strip()
2424 commitFound = True
2425 # Convert from p4 time format
2426 try:
2427 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2428 except ValueError:
2429 print "Could not convert label time %s" % labelDetails['Update']
2430 tmwhen = 1
2432 when = int(time.mktime(tmwhen))
2433 self.streamTag(stream, name, labelDetails, gitCommit, when)
2434 if verbose:
2435 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2436 else:
2437 if verbose:
2438 print "Label %s has no changelists - possibly deleted?" % name
2440 if not commitFound:
2441 # We can't import this label; don't try again as it will get very
2442 # expensive repeatedly fetching all the files for labels that will
2443 # never be imported. If the label is moved in the future, the
2444 # ignore will need to be removed manually.
2445 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2447 def guessProjectName(self):
2448 for p in self.depotPaths:
2449 if p.endswith("/"):
2450 p = p[:-1]
2451 p = p[p.strip().rfind("/") + 1:]
2452 if not p.endswith("/"):
2453 p += "/"
2454 return p
2456 def getBranchMapping(self):
2457 lostAndFoundBranches = set()
2459 user = gitConfig("git-p4.branchUser")
2460 if len(user) > 0:
2461 command = "branches -u %s" % user
2462 else:
2463 command = "branches"
2465 for info in p4CmdList(command):
2466 details = p4Cmd(["branch", "-o", info["branch"]])
2467 viewIdx = 0
2468 while details.has_key("View%s" % viewIdx):
2469 paths = details["View%s" % viewIdx].split(" ")
2470 viewIdx = viewIdx + 1
2471 # require standard //depot/foo/... //depot/bar/... mapping
2472 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2473 continue
2474 source = paths[0]
2475 destination = paths[1]
2476 ## HACK
2477 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2478 source = source[len(self.depotPaths[0]):-4]
2479 destination = destination[len(self.depotPaths[0]):-4]
2481 if destination in self.knownBranches:
2482 if not self.silent:
2483 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2484 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2485 continue
2487 self.knownBranches[destination] = source
2489 lostAndFoundBranches.discard(destination)
2491 if source not in self.knownBranches:
2492 lostAndFoundBranches.add(source)
2494 # Perforce does not strictly require branches to be defined, so we also
2495 # check git config for a branch list.
2497 # Example of branch definition in git config file:
2498 # [git-p4]
2499 # branchList=main:branchA
2500 # branchList=main:branchB
2501 # branchList=branchA:branchC
2502 configBranches = gitConfigList("git-p4.branchList")
2503 for branch in configBranches:
2504 if branch:
2505 (source, destination) = branch.split(":")
2506 self.knownBranches[destination] = source
2508 lostAndFoundBranches.discard(destination)
2510 if source not in self.knownBranches:
2511 lostAndFoundBranches.add(source)
2514 for branch in lostAndFoundBranches:
2515 self.knownBranches[branch] = branch
2517 def getBranchMappingFromGitBranches(self):
2518 branches = p4BranchesInGit(self.importIntoRemotes)
2519 for branch in branches.keys():
2520 if branch == "master":
2521 branch = "main"
2522 else:
2523 branch = branch[len(self.projectName):]
2524 self.knownBranches[branch] = branch
2526 def listExistingP4GitBranches(self):
2527 # branches holds mapping from name to commit
2528 branches = p4BranchesInGit(self.importIntoRemotes)
2529 self.p4BranchesInGit = branches.keys()
2530 for branch in branches.keys():
2531 self.initialParents[self.refPrefix + branch] = branches[branch]
2533 def updateOptionDict(self, d):
2534 option_keys = {}
2535 if self.keepRepoPath:
2536 option_keys['keepRepoPath'] = 1
2538 d["options"] = ' '.join(sorted(option_keys.keys()))
2540 def readOptions(self, d):
2541 self.keepRepoPath = (d.has_key('options')
2542 and ('keepRepoPath' in d['options']))
2544 def gitRefForBranch(self, branch):
2545 if branch == "main":
2546 return self.refPrefix + "master"
2548 if len(branch) <= 0:
2549 return branch
2551 return self.refPrefix + self.projectName + branch
2553 def gitCommitByP4Change(self, ref, change):
2554 if self.verbose:
2555 print "looking in ref " + ref + " for change %s using bisect..." % change
2557 earliestCommit = ""
2558 latestCommit = parseRevision(ref)
2560 while True:
2561 if self.verbose:
2562 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2563 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2564 if len(next) == 0:
2565 if self.verbose:
2566 print "argh"
2567 return ""
2568 log = extractLogMessageFromGitCommit(next)
2569 settings = extractSettingsGitLog(log)
2570 currentChange = int(settings['change'])
2571 if self.verbose:
2572 print "current change %s" % currentChange
2574 if currentChange == change:
2575 if self.verbose:
2576 print "found %s" % next
2577 return next
2579 if currentChange < change:
2580 earliestCommit = "^%s" % next
2581 else:
2582 latestCommit = "%s" % next
2584 return ""
2586 def importNewBranch(self, branch, maxChange):
2587 # make fast-import flush all changes to disk and update the refs using the checkpoint
2588 # command so that we can try to find the branch parent in the git history
2589 self.gitStream.write("checkpoint\n\n");
2590 self.gitStream.flush();
2591 branchPrefix = self.depotPaths[0] + branch + "/"
2592 range = "@1,%s" % maxChange
2593 #print "prefix" + branchPrefix
2594 changes = p4ChangesForPaths([branchPrefix], range)
2595 if len(changes) <= 0:
2596 return False
2597 firstChange = changes[0]
2598 #print "first change in branch: %s" % firstChange
2599 sourceBranch = self.knownBranches[branch]
2600 sourceDepotPath = self.depotPaths[0] + sourceBranch
2601 sourceRef = self.gitRefForBranch(sourceBranch)
2602 #print "source " + sourceBranch
2604 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2605 #print "branch parent: %s" % branchParentChange
2606 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2607 if len(gitParent) > 0:
2608 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2609 #print "parent git commit: %s" % gitParent
2611 self.importChanges(changes)
2612 return True
2614 def searchParent(self, parent, branch, target):
2615 parentFound = False
2616 for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2617 blob = blob.strip()
2618 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2619 parentFound = True
2620 if self.verbose:
2621 print "Found parent of %s in commit %s" % (branch, blob)
2622 break
2623 if parentFound:
2624 return blob
2625 else:
2626 return None
2628 def importChanges(self, changes):
2629 cnt = 1
2630 for change in changes:
2631 description = p4_describe(change)
2632 self.updateOptionDict(description)
2634 if not self.silent:
2635 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2636 sys.stdout.flush()
2637 cnt = cnt + 1
2639 try:
2640 if self.detectBranches:
2641 branches = self.splitFilesIntoBranches(description)
2642 for branch in branches.keys():
2643 ## HACK --hwn
2644 branchPrefix = self.depotPaths[0] + branch + "/"
2645 self.branchPrefixes = [ branchPrefix ]
2647 parent = ""
2649 filesForCommit = branches[branch]
2651 if self.verbose:
2652 print "branch is %s" % branch
2654 self.updatedBranches.add(branch)
2656 if branch not in self.createdBranches:
2657 self.createdBranches.add(branch)
2658 parent = self.knownBranches[branch]
2659 if parent == branch:
2660 parent = ""
2661 else:
2662 fullBranch = self.projectName + branch
2663 if fullBranch not in self.p4BranchesInGit:
2664 if not self.silent:
2665 print("\n Importing new branch %s" % fullBranch);
2666 if self.importNewBranch(branch, change - 1):
2667 parent = ""
2668 self.p4BranchesInGit.append(fullBranch)
2669 if not self.silent:
2670 print("\n Resuming with change %s" % change);
2672 if self.verbose:
2673 print "parent determined through known branches: %s" % parent
2675 branch = self.gitRefForBranch(branch)
2676 parent = self.gitRefForBranch(parent)
2678 if self.verbose:
2679 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2681 if len(parent) == 0 and branch in self.initialParents:
2682 parent = self.initialParents[branch]
2683 del self.initialParents[branch]
2685 blob = None
2686 if len(parent) > 0:
2687 tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2688 if self.verbose:
2689 print "Creating temporary branch: " + tempBranch
2690 self.commit(description, filesForCommit, tempBranch)
2691 self.tempBranches.append(tempBranch)
2692 self.checkpoint()
2693 blob = self.searchParent(parent, branch, tempBranch)
2694 if blob:
2695 self.commit(description, filesForCommit, branch, blob)
2696 else:
2697 if self.verbose:
2698 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2699 self.commit(description, filesForCommit, branch, parent)
2700 else:
2701 files = self.extractFilesFromCommit(description)
2702 self.commit(description, files, self.branch,
2703 self.initialParent)
2704 self.initialParent = ""
2705 except IOError:
2706 print self.gitError.read()
2707 sys.exit(1)
2709 def importHeadRevision(self, revision):
2710 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2712 details = {}
2713 details["user"] = "git perforce import user"
2714 details["desc"] = ("Initial import of %s from the state at revision %s\n"
2715 % (' '.join(self.depotPaths), revision))
2716 details["change"] = revision
2717 newestRevision = 0
2719 fileCnt = 0
2720 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2722 for info in p4CmdList(["files"] + fileArgs):
2724 if 'code' in info and info['code'] == 'error':
2725 sys.stderr.write("p4 returned an error: %s\n"
2726 % info['data'])
2727 if info['data'].find("must refer to client") >= 0:
2728 sys.stderr.write("This particular p4 error is misleading.\n")
2729 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2730 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
2731 sys.exit(1)
2732 if 'p4ExitCode' in info:
2733 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2734 sys.exit(1)
2737 change = int(info["change"])
2738 if change > newestRevision:
2739 newestRevision = change
2741 if info["action"] in self.delete_actions:
2742 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2743 #fileCnt = fileCnt + 1
2744 continue
2746 for prop in ["depotFile", "rev", "action", "type" ]:
2747 details["%s%s" % (prop, fileCnt)] = info[prop]
2749 fileCnt = fileCnt + 1
2751 details["change"] = newestRevision
2753 # Use time from top-most change so that all git p4 clones of
2754 # the same p4 repo have the same commit SHA1s.
2755 res = p4_describe(newestRevision)
2756 details["time"] = res["time"]
2758 self.updateOptionDict(details)
2759 try:
2760 self.commit(details, self.extractFilesFromCommit(details), self.branch)
2761 except IOError:
2762 print "IO error with git fast-import. Is your git version recent enough?"
2763 print self.gitError.read()
2766 def run(self, args):
2767 self.depotPaths = []
2768 self.changeRange = ""
2769 self.initialParent = ""
2770 self.previousDepotPaths = []
2772 # map from branch depot path to parent branch
2773 self.knownBranches = {}
2774 self.initialParents = {}
2775 self.hasOrigin = originP4BranchesExist()
2776 if not self.syncWithOrigin:
2777 self.hasOrigin = False
2779 if self.importIntoRemotes:
2780 self.refPrefix = "refs/remotes/p4/"
2781 else:
2782 self.refPrefix = "refs/heads/p4/"
2784 if self.syncWithOrigin and self.hasOrigin:
2785 if not self.silent:
2786 print "Syncing with origin first by calling git fetch origin"
2787 system("git fetch origin")
2789 if len(self.branch) == 0:
2790 self.branch = self.refPrefix + "master"
2791 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2792 system("git update-ref %s refs/heads/p4" % self.branch)
2793 system("git branch -D p4");
2794 # create it /after/ importing, when master exists
2795 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2796 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2798 # accept either the command-line option, or the configuration variable
2799 if self.useClientSpec:
2800 # will use this after clone to set the variable
2801 self.useClientSpec_from_options = True
2802 else:
2803 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2804 self.useClientSpec = True
2805 if self.useClientSpec:
2806 self.clientSpecDirs = getClientSpec()
2808 # TODO: should always look at previous commits,
2809 # merge with previous imports, if possible.
2810 if args == []:
2811 if self.hasOrigin:
2812 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2813 self.listExistingP4GitBranches()
2815 if len(self.p4BranchesInGit) > 1:
2816 if not self.silent:
2817 print "Importing from/into multiple branches"
2818 self.detectBranches = True
2820 if self.verbose:
2821 print "branches: %s" % self.p4BranchesInGit
2823 p4Change = 0
2824 for branch in self.p4BranchesInGit:
2825 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
2827 settings = extractSettingsGitLog(logMsg)
2829 self.readOptions(settings)
2830 if (settings.has_key('depot-paths')
2831 and settings.has_key ('change')):
2832 change = int(settings['change']) + 1
2833 p4Change = max(p4Change, change)
2835 depotPaths = sorted(settings['depot-paths'])
2836 if self.previousDepotPaths == []:
2837 self.previousDepotPaths = depotPaths
2838 else:
2839 paths = []
2840 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2841 prev_list = prev.split("/")
2842 cur_list = cur.split("/")
2843 for i in range(0, min(len(cur_list), len(prev_list))):
2844 if cur_list[i] <> prev_list[i]:
2845 i = i - 1
2846 break
2848 paths.append ("/".join(cur_list[:i + 1]))
2850 self.previousDepotPaths = paths
2852 if p4Change > 0:
2853 self.depotPaths = sorted(self.previousDepotPaths)
2854 self.changeRange = "@%s,#head" % p4Change
2855 if not self.detectBranches:
2856 self.initialParent = parseRevision(self.branch)
2857 if not self.silent and not self.detectBranches:
2858 print "Performing incremental import into %s git branch" % self.branch
2860 if not self.branch.startswith("refs/"):
2861 self.branch = "refs/heads/" + self.branch
2863 if len(args) == 0 and self.depotPaths:
2864 if not self.silent:
2865 print "Depot paths: %s" % ' '.join(self.depotPaths)
2866 else:
2867 if self.depotPaths and self.depotPaths != args:
2868 print ("previous import used depot path %s and now %s was specified. "
2869 "This doesn't work!" % (' '.join (self.depotPaths),
2870 ' '.join (args)))
2871 sys.exit(1)
2873 self.depotPaths = sorted(args)
2875 revision = ""
2876 self.users = {}
2878 # Make sure no revision specifiers are used when --changesfile
2879 # is specified.
2880 bad_changesfile = False
2881 if len(self.changesFile) > 0:
2882 for p in self.depotPaths:
2883 if p.find("@") >= 0 or p.find("#") >= 0:
2884 bad_changesfile = True
2885 break
2886 if bad_changesfile:
2887 die("Option --changesfile is incompatible with revision specifiers")
2889 newPaths = []
2890 for p in self.depotPaths:
2891 if p.find("@") != -1:
2892 atIdx = p.index("@")
2893 self.changeRange = p[atIdx:]
2894 if self.changeRange == "@all":
2895 self.changeRange = ""
2896 elif ',' not in self.changeRange:
2897 revision = self.changeRange
2898 self.changeRange = ""
2899 p = p[:atIdx]
2900 elif p.find("#") != -1:
2901 hashIdx = p.index("#")
2902 revision = p[hashIdx:]
2903 p = p[:hashIdx]
2904 elif self.previousDepotPaths == []:
2905 # pay attention to changesfile, if given, else import
2906 # the entire p4 tree at the head revision
2907 if len(self.changesFile) == 0:
2908 revision = "#head"
2910 p = re.sub ("\.\.\.$", "", p)
2911 if not p.endswith("/"):
2912 p += "/"
2914 newPaths.append(p)
2916 self.depotPaths = newPaths
2918 # --detect-branches may change this for each branch
2919 self.branchPrefixes = self.depotPaths
2921 self.loadUserMapFromCache()
2922 self.labels = {}
2923 if self.detectLabels:
2924 self.getLabels();
2926 if self.detectBranches:
2927 ## FIXME - what's a P4 projectName ?
2928 self.projectName = self.guessProjectName()
2930 if self.hasOrigin:
2931 self.getBranchMappingFromGitBranches()
2932 else:
2933 self.getBranchMapping()
2934 if self.verbose:
2935 print "p4-git branches: %s" % self.p4BranchesInGit
2936 print "initial parents: %s" % self.initialParents
2937 for b in self.p4BranchesInGit:
2938 if b != "master":
2940 ## FIXME
2941 b = b[len(self.projectName):]
2942 self.createdBranches.add(b)
2944 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2946 self.importProcess = subprocess.Popen(["git", "fast-import"],
2947 stdin=subprocess.PIPE,
2948 stdout=subprocess.PIPE,
2949 stderr=subprocess.PIPE);
2950 self.gitOutput = self.importProcess.stdout
2951 self.gitStream = self.importProcess.stdin
2952 self.gitError = self.importProcess.stderr
2954 if revision:
2955 self.importHeadRevision(revision)
2956 else:
2957 changes = []
2959 if len(self.changesFile) > 0:
2960 output = open(self.changesFile).readlines()
2961 changeSet = set()
2962 for line in output:
2963 changeSet.add(int(line))
2965 for change in changeSet:
2966 changes.append(change)
2968 changes.sort()
2969 else:
2970 # catch "git p4 sync" with no new branches, in a repo that
2971 # does not have any existing p4 branches
2972 if len(args) == 0 and not self.p4BranchesInGit:
2973 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2974 if self.verbose:
2975 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2976 self.changeRange)
2977 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2979 if len(self.maxChanges) > 0:
2980 changes = changes[:min(int(self.maxChanges), len(changes))]
2982 if len(changes) == 0:
2983 if not self.silent:
2984 print "No changes to import!"
2985 else:
2986 if not self.silent and not self.detectBranches:
2987 print "Import destination: %s" % self.branch
2989 self.updatedBranches = set()
2991 self.importChanges(changes)
2993 if not self.silent:
2994 print ""
2995 if len(self.updatedBranches) > 0:
2996 sys.stdout.write("Updated branches: ")
2997 for b in self.updatedBranches:
2998 sys.stdout.write("%s " % b)
2999 sys.stdout.write("\n")
3001 if gitConfig("git-p4.importLabels", "--bool") == "true":
3002 self.importLabels = True
3004 if self.importLabels:
3005 p4Labels = getP4Labels(self.depotPaths)
3006 gitTags = getGitTags()
3008 missingP4Labels = p4Labels - gitTags
3009 self.importP4Labels(self.gitStream, missingP4Labels)
3011 self.gitStream.close()
3012 if self.importProcess.wait() != 0:
3013 die("fast-import failed: %s" % self.gitError.read())
3014 self.gitOutput.close()
3015 self.gitError.close()
3017 # Cleanup temporary branches created during import
3018 if self.tempBranches != []:
3019 for branch in self.tempBranches:
3020 read_pipe("git update-ref -d %s" % branch)
3021 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3023 return True
3025 class P4Rebase(Command):
3026 def __init__(self):
3027 Command.__init__(self)
3028 self.options = [
3029 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3031 self.importLabels = False
3032 self.description = ("Fetches the latest revision from perforce and "
3033 + "rebases the current work (branch) against it")
3035 def run(self, args):
3036 sync = P4Sync()
3037 sync.importLabels = self.importLabels
3038 sync.run([])
3040 return self.rebase()
3042 def rebase(self):
3043 if os.system("git update-index --refresh") != 0:
3044 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3045 if len(read_pipe("git diff-index HEAD --")) > 0:
3046 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
3048 [upstream, settings] = findUpstreamBranchPoint()
3049 if len(upstream) == 0:
3050 die("Cannot find upstream branchpoint for rebase")
3052 # the branchpoint may be p4/foo~3, so strip off the parent
3053 upstream = re.sub("~[0-9]+$", "", upstream)
3055 print "Rebasing the current branch onto %s" % upstream
3056 oldHead = read_pipe("git rev-parse HEAD").strip()
3057 system("git rebase %s" % upstream)
3058 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3059 return True
3061 class P4Clone(P4Sync):
3062 def __init__(self):
3063 P4Sync.__init__(self)
3064 self.description = "Creates a new git repository and imports from Perforce into it"
3065 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3066 self.options += [
3067 optparse.make_option("--destination", dest="cloneDestination",
3068 action='store', default=None,
3069 help="where to leave result of the clone"),
3070 optparse.make_option("-/", dest="cloneExclude",
3071 action="append", type="string",
3072 help="exclude depot path"),
3073 optparse.make_option("--bare", dest="cloneBare",
3074 action="store_true", default=False),
3076 self.cloneDestination = None
3077 self.needsGit = False
3078 self.cloneBare = False
3080 # This is required for the "append" cloneExclude action
3081 def ensure_value(self, attr, value):
3082 if not hasattr(self, attr) or getattr(self, attr) is None:
3083 setattr(self, attr, value)
3084 return getattr(self, attr)
3086 def defaultDestination(self, args):
3087 ## TODO: use common prefix of args?
3088 depotPath = args[0]
3089 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3090 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3091 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3092 depotDir = re.sub(r"/$", "", depotDir)
3093 return os.path.split(depotDir)[1]
3095 def run(self, args):
3096 if len(args) < 1:
3097 return False
3099 if self.keepRepoPath and not self.cloneDestination:
3100 sys.stderr.write("Must specify destination for --keep-path\n")
3101 sys.exit(1)
3103 depotPaths = args
3105 if not self.cloneDestination and len(depotPaths) > 1:
3106 self.cloneDestination = depotPaths[-1]
3107 depotPaths = depotPaths[:-1]
3109 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3110 for p in depotPaths:
3111 if not p.startswith("//"):
3112 return False
3114 if not self.cloneDestination:
3115 self.cloneDestination = self.defaultDestination(args)
3117 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3119 if not os.path.exists(self.cloneDestination):
3120 os.makedirs(self.cloneDestination)
3121 chdir(self.cloneDestination)
3123 init_cmd = [ "git", "init" ]
3124 if self.cloneBare:
3125 init_cmd.append("--bare")
3126 retcode = subprocess.call(init_cmd)
3127 if retcode:
3128 raise CalledProcessError(retcode, init_cmd)
3130 if not P4Sync.run(self, depotPaths):
3131 return False
3132 if self.branch != "master":
3133 if self.importIntoRemotes:
3134 masterbranch = "refs/remotes/p4/master"
3135 else:
3136 masterbranch = "refs/heads/p4/master"
3137 if gitBranchExists(masterbranch):
3138 system("git branch master %s" % masterbranch)
3139 if not self.cloneBare:
3140 system("git checkout -f")
3141 else:
3142 print "Could not detect main branch. No checkout/master branch created."
3144 # auto-set this variable if invoked with --use-client-spec
3145 if self.useClientSpec_from_options:
3146 system("git config --bool git-p4.useclientspec true")
3148 return True
3150 class P4Branches(Command):
3151 def __init__(self):
3152 Command.__init__(self)
3153 self.options = [ ]
3154 self.description = ("Shows the git branches that hold imports and their "
3155 + "corresponding perforce depot paths")
3156 self.verbose = False
3158 def run(self, args):
3159 if originP4BranchesExist():
3160 createOrUpdateBranchesFromOrigin()
3162 cmdline = "git rev-parse --symbolic "
3163 cmdline += " --remotes"
3165 for line in read_pipe_lines(cmdline):
3166 line = line.strip()
3168 if not line.startswith('p4/') or line == "p4/HEAD":
3169 continue
3170 branch = line
3172 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3173 settings = extractSettingsGitLog(log)
3175 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3176 return True
3178 class HelpFormatter(optparse.IndentedHelpFormatter):
3179 def __init__(self):
3180 optparse.IndentedHelpFormatter.__init__(self)
3182 def format_description(self, description):
3183 if description:
3184 return description + "\n"
3185 else:
3186 return ""
3188 def printUsage(commands):
3189 print "usage: %s <command> [options]" % sys.argv[0]
3190 print ""
3191 print "valid commands: %s" % ", ".join(commands)
3192 print ""
3193 print "Try %s <command> --help for command specific help." % sys.argv[0]
3194 print ""
3196 commands = {
3197 "debug" : P4Debug,
3198 "submit" : P4Submit,
3199 "commit" : P4Submit,
3200 "sync" : P4Sync,
3201 "rebase" : P4Rebase,
3202 "clone" : P4Clone,
3203 "rollback" : P4RollBack,
3204 "branches" : P4Branches
3208 def main():
3209 if len(sys.argv[1:]) == 0:
3210 printUsage(commands.keys())
3211 sys.exit(2)
3213 cmdName = sys.argv[1]
3214 try:
3215 klass = commands[cmdName]
3216 cmd = klass()
3217 except KeyError:
3218 print "unknown command %s" % cmdName
3219 print ""
3220 printUsage(commands.keys())
3221 sys.exit(2)
3223 options = cmd.options
3224 cmd.gitdir = os.environ.get("GIT_DIR", None)
3226 args = sys.argv[2:]
3228 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3229 if cmd.needsGit:
3230 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3232 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3233 options,
3234 description = cmd.description,
3235 formatter = HelpFormatter())
3237 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3238 global verbose
3239 verbose = cmd.verbose
3240 if cmd.needsGit:
3241 if cmd.gitdir == None:
3242 cmd.gitdir = os.path.abspath(".git")
3243 if not isValidGitDir(cmd.gitdir):
3244 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3245 if os.path.exists(cmd.gitdir):
3246 cdup = read_pipe("git rev-parse --show-cdup").strip()
3247 if len(cdup) > 0:
3248 chdir(cdup);
3250 if not isValidGitDir(cmd.gitdir):
3251 if isValidGitDir(cmd.gitdir + "/.git"):
3252 cmd.gitdir += "/.git"
3253 else:
3254 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3256 os.environ["GIT_DIR"] = cmd.gitdir
3258 if not cmd.run(args):
3259 parser.print_help()
3260 sys.exit(2)
3263 if __name__ == '__main__':
3264 main()