Merge branch 'jc/maint-push-refspec-default-doc' into next
[git/mjg.git] / git-p4.py
blob647f11020257598a8245eb1e84f64b80a2c55aca
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
26 try:
27 from subprocess import CalledProcessError
28 except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
41 verbose = False
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
46 def p4_build_cmd(cmd):
47 """Build a suitable p4 command line.
49 This consolidates building and returning a p4 command line into one
50 location. It means that hooking into the environment, or other configuration
51 can be done more easily.
52 """
53 real_cmd = ["p4"]
55 user = gitConfig("git-p4.user")
56 if len(user) > 0:
57 real_cmd += ["-u",user]
59 password = gitConfig("git-p4.password")
60 if len(password) > 0:
61 real_cmd += ["-P", password]
63 port = gitConfig("git-p4.port")
64 if len(port) > 0:
65 real_cmd += ["-p", port]
67 host = gitConfig("git-p4.host")
68 if len(host) > 0:
69 real_cmd += ["-H", host]
71 client = gitConfig("git-p4.client")
72 if len(client) > 0:
73 real_cmd += ["-c", client]
76 if isinstance(cmd,basestring):
77 real_cmd = ' '.join(real_cmd) + ' ' + cmd
78 else:
79 real_cmd += cmd
80 return real_cmd
82 def chdir(dir):
83 # P4 uses the PWD environment variable rather than getcwd(). Since we're
84 # not using the shell, we have to set it ourselves. This path could
85 # be relative, so go there first, then figure out where we ended up.
86 os.chdir(dir)
87 os.environ['PWD'] = os.getcwd()
89 def die(msg):
90 if verbose:
91 raise Exception(msg)
92 else:
93 sys.stderr.write(msg + "\n")
94 sys.exit(1)
96 def write_pipe(c, stdin):
97 if verbose:
98 sys.stderr.write('Writing pipe: %s\n' % str(c))
100 expand = isinstance(c,basestring)
101 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
102 pipe = p.stdin
103 val = pipe.write(stdin)
104 pipe.close()
105 if p.wait():
106 die('Command failed: %s' % str(c))
108 return val
110 def p4_write_pipe(c, stdin):
111 real_cmd = p4_build_cmd(c)
112 return write_pipe(real_cmd, stdin)
114 def read_pipe(c, ignore_error=False):
115 if verbose:
116 sys.stderr.write('Reading pipe: %s\n' % str(c))
118 expand = isinstance(c,basestring)
119 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
120 pipe = p.stdout
121 val = pipe.read()
122 if p.wait() and not ignore_error:
123 die('Command failed: %s' % str(c))
125 return val
127 def p4_read_pipe(c, ignore_error=False):
128 real_cmd = p4_build_cmd(c)
129 return read_pipe(real_cmd, ignore_error)
131 def read_pipe_lines(c):
132 if verbose:
133 sys.stderr.write('Reading pipe: %s\n' % str(c))
135 expand = isinstance(c, basestring)
136 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
137 pipe = p.stdout
138 val = pipe.readlines()
139 if pipe.close() or p.wait():
140 die('Command failed: %s' % str(c))
142 return val
144 def p4_read_pipe_lines(c):
145 """Specifically invoke p4 on the command supplied. """
146 real_cmd = p4_build_cmd(c)
147 return read_pipe_lines(real_cmd)
149 def p4_has_command(cmd):
150 """Ask p4 for help on this command. If it returns an error, the
151 command does not exist in this version of p4."""
152 real_cmd = p4_build_cmd(["help", cmd])
153 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
154 stderr=subprocess.PIPE)
155 p.communicate()
156 return p.returncode == 0
158 def p4_has_move_command():
159 """See if the move command exists, that it supports -k, and that
160 it has not been administratively disabled. The arguments
161 must be correct, but the filenames do not have to exist. Use
162 ones with wildcards so even if they exist, it will fail."""
164 if not p4_has_command("move"):
165 return False
166 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
167 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
168 (out, err) = p.communicate()
169 # return code will be 1 in either case
170 if err.find("Invalid option") >= 0:
171 return False
172 if err.find("disabled") >= 0:
173 return False
174 # assume it failed because @... was invalid changelist
175 return True
177 def system(cmd):
178 expand = isinstance(cmd,basestring)
179 if verbose:
180 sys.stderr.write("executing %s\n" % str(cmd))
181 retcode = subprocess.call(cmd, shell=expand)
182 if retcode:
183 raise CalledProcessError(retcode, cmd)
185 def p4_system(cmd):
186 """Specifically invoke p4 as the system command. """
187 real_cmd = p4_build_cmd(cmd)
188 expand = isinstance(real_cmd, basestring)
189 retcode = subprocess.call(real_cmd, shell=expand)
190 if retcode:
191 raise CalledProcessError(retcode, real_cmd)
193 _p4_version_string = None
194 def p4_version_string():
195 """Read the version string, showing just the last line, which
196 hopefully is the interesting version bit.
198 $ p4 -V
199 Perforce - The Fast Software Configuration Management System.
200 Copyright 1995-2011 Perforce Software. All rights reserved.
201 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
203 global _p4_version_string
204 if not _p4_version_string:
205 a = p4_read_pipe_lines(["-V"])
206 _p4_version_string = a[-1].rstrip()
207 return _p4_version_string
209 def p4_integrate(src, dest):
210 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
212 def p4_sync(f, *options):
213 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
215 def p4_add(f):
216 # forcibly add file names with wildcards
217 if wildcard_present(f):
218 p4_system(["add", "-f", f])
219 else:
220 p4_system(["add", f])
222 def p4_delete(f):
223 p4_system(["delete", wildcard_encode(f)])
225 def p4_edit(f):
226 p4_system(["edit", wildcard_encode(f)])
228 def p4_revert(f):
229 p4_system(["revert", wildcard_encode(f)])
231 def p4_reopen(type, f):
232 p4_system(["reopen", "-t", type, wildcard_encode(f)])
234 def p4_move(src, dest):
235 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
237 def p4_describe(change):
238 """Make sure it returns a valid result by checking for
239 the presence of field "time". Return a dict of the
240 results."""
242 ds = p4CmdList(["describe", "-s", str(change)])
243 if len(ds) != 1:
244 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
246 d = ds[0]
248 if "p4ExitCode" in d:
249 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
250 str(d)))
251 if "code" in d:
252 if d["code"] == "error":
253 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
255 if "time" not in d:
256 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
258 return d
261 # Canonicalize the p4 type and return a tuple of the
262 # base type, plus any modifiers. See "p4 help filetypes"
263 # for a list and explanation.
265 def split_p4_type(p4type):
267 p4_filetypes_historical = {
268 "ctempobj": "binary+Sw",
269 "ctext": "text+C",
270 "cxtext": "text+Cx",
271 "ktext": "text+k",
272 "kxtext": "text+kx",
273 "ltext": "text+F",
274 "tempobj": "binary+FSw",
275 "ubinary": "binary+F",
276 "uresource": "resource+F",
277 "uxbinary": "binary+Fx",
278 "xbinary": "binary+x",
279 "xltext": "text+Fx",
280 "xtempobj": "binary+Swx",
281 "xtext": "text+x",
282 "xunicode": "unicode+x",
283 "xutf16": "utf16+x",
285 if p4type in p4_filetypes_historical:
286 p4type = p4_filetypes_historical[p4type]
287 mods = ""
288 s = p4type.split("+")
289 base = s[0]
290 mods = ""
291 if len(s) > 1:
292 mods = s[1]
293 return (base, mods)
296 # return the raw p4 type of a file (text, text+ko, etc)
298 def p4_type(file):
299 results = p4CmdList(["fstat", "-T", "headType", file])
300 return results[0]['headType']
303 # Given a type base and modifier, return a regexp matching
304 # the keywords that can be expanded in the file
306 def p4_keywords_regexp_for_type(base, type_mods):
307 if base in ("text", "unicode", "binary"):
308 kwords = None
309 if "ko" in type_mods:
310 kwords = 'Id|Header'
311 elif "k" in type_mods:
312 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
313 else:
314 return None
315 pattern = r"""
316 \$ # Starts with a dollar, followed by...
317 (%s) # one of the keywords, followed by...
318 (:[^$\n]+)? # possibly an old expansion, followed by...
319 \$ # another dollar
320 """ % kwords
321 return pattern
322 else:
323 return None
326 # Given a file, return a regexp matching the possible
327 # RCS keywords that will be expanded, or None for files
328 # with kw expansion turned off.
330 def p4_keywords_regexp_for_file(file):
331 if not os.path.exists(file):
332 return None
333 else:
334 (type_base, type_mods) = split_p4_type(p4_type(file))
335 return p4_keywords_regexp_for_type(type_base, type_mods)
337 def setP4ExecBit(file, mode):
338 # Reopens an already open file and changes the execute bit to match
339 # the execute bit setting in the passed in mode.
341 p4Type = "+x"
343 if not isModeExec(mode):
344 p4Type = getP4OpenedType(file)
345 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
346 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
347 if p4Type[-1] == "+":
348 p4Type = p4Type[0:-1]
350 p4_reopen(p4Type, file)
352 def getP4OpenedType(file):
353 # Returns the perforce file type for the given file.
355 result = p4_read_pipe(["opened", wildcard_encode(file)])
356 match = re.match(".*\((.+)\)\r?$", result)
357 if match:
358 return match.group(1)
359 else:
360 die("Could not determine file type for %s (result: '%s')" % (file, result))
362 # Return the set of all p4 labels
363 def getP4Labels(depotPaths):
364 labels = set()
365 if isinstance(depotPaths,basestring):
366 depotPaths = [depotPaths]
368 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
369 label = l['label']
370 labels.add(label)
372 return labels
374 # Return the set of all git tags
375 def getGitTags():
376 gitTags = set()
377 for line in read_pipe_lines(["git", "tag"]):
378 tag = line.strip()
379 gitTags.add(tag)
380 return gitTags
382 def diffTreePattern():
383 # This is a simple generator for the diff tree regex pattern. This could be
384 # a class variable if this and parseDiffTreeEntry were a part of a class.
385 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
386 while True:
387 yield pattern
389 def parseDiffTreeEntry(entry):
390 """Parses a single diff tree entry into its component elements.
392 See git-diff-tree(1) manpage for details about the format of the diff
393 output. This method returns a dictionary with the following elements:
395 src_mode - The mode of the source file
396 dst_mode - The mode of the destination file
397 src_sha1 - The sha1 for the source file
398 dst_sha1 - The sha1 fr the destination file
399 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
400 status_score - The score for the status (applicable for 'C' and 'R'
401 statuses). This is None if there is no score.
402 src - The path for the source file.
403 dst - The path for the destination file. This is only present for
404 copy or renames. If it is not present, this is None.
406 If the pattern is not matched, None is returned."""
408 match = diffTreePattern().next().match(entry)
409 if match:
410 return {
411 'src_mode': match.group(1),
412 'dst_mode': match.group(2),
413 'src_sha1': match.group(3),
414 'dst_sha1': match.group(4),
415 'status': match.group(5),
416 'status_score': match.group(6),
417 'src': match.group(7),
418 'dst': match.group(10)
420 return None
422 def isModeExec(mode):
423 # Returns True if the given git mode represents an executable file,
424 # otherwise False.
425 return mode[-3:] == "755"
427 def isModeExecChanged(src_mode, dst_mode):
428 return isModeExec(src_mode) != isModeExec(dst_mode)
430 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
432 if isinstance(cmd,basestring):
433 cmd = "-G " + cmd
434 expand = True
435 else:
436 cmd = ["-G"] + cmd
437 expand = False
439 cmd = p4_build_cmd(cmd)
440 if verbose:
441 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
443 # Use a temporary file to avoid deadlocks without
444 # subprocess.communicate(), which would put another copy
445 # of stdout into memory.
446 stdin_file = None
447 if stdin is not None:
448 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
449 if isinstance(stdin,basestring):
450 stdin_file.write(stdin)
451 else:
452 for i in stdin:
453 stdin_file.write(i + '\n')
454 stdin_file.flush()
455 stdin_file.seek(0)
457 p4 = subprocess.Popen(cmd,
458 shell=expand,
459 stdin=stdin_file,
460 stdout=subprocess.PIPE)
462 result = []
463 try:
464 while True:
465 entry = marshal.load(p4.stdout)
466 if cb is not None:
467 cb(entry)
468 else:
469 result.append(entry)
470 except EOFError:
471 pass
472 exitCode = p4.wait()
473 if exitCode != 0:
474 entry = {}
475 entry["p4ExitCode"] = exitCode
476 result.append(entry)
478 return result
480 def p4Cmd(cmd):
481 list = p4CmdList(cmd)
482 result = {}
483 for entry in list:
484 result.update(entry)
485 return result;
487 def p4Where(depotPath):
488 if not depotPath.endswith("/"):
489 depotPath += "/"
490 depotPath = depotPath + "..."
491 outputList = p4CmdList(["where", depotPath])
492 output = None
493 for entry in outputList:
494 if "depotFile" in entry:
495 if entry["depotFile"] == depotPath:
496 output = entry
497 break
498 elif "data" in entry:
499 data = entry.get("data")
500 space = data.find(" ")
501 if data[:space] == depotPath:
502 output = entry
503 break
504 if output == None:
505 return ""
506 if output["code"] == "error":
507 return ""
508 clientPath = ""
509 if "path" in output:
510 clientPath = output.get("path")
511 elif "data" in output:
512 data = output.get("data")
513 lastSpace = data.rfind(" ")
514 clientPath = data[lastSpace + 1:]
516 if clientPath.endswith("..."):
517 clientPath = clientPath[:-3]
518 return clientPath
520 def currentGitBranch():
521 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
523 def isValidGitDir(path):
524 if (os.path.exists(path + "/HEAD")
525 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
526 return True;
527 return False
529 def parseRevision(ref):
530 return read_pipe("git rev-parse %s" % ref).strip()
532 def branchExists(ref):
533 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
534 ignore_error=True)
535 return len(rev) > 0
537 def extractLogMessageFromGitCommit(commit):
538 logMessage = ""
540 ## fixme: title is first line of commit, not 1st paragraph.
541 foundTitle = False
542 for log in read_pipe_lines("git cat-file commit %s" % commit):
543 if not foundTitle:
544 if len(log) == 1:
545 foundTitle = True
546 continue
548 logMessage += log
549 return logMessage
551 def extractSettingsGitLog(log):
552 values = {}
553 for line in log.split("\n"):
554 line = line.strip()
555 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
556 if not m:
557 continue
559 assignments = m.group(1).split (':')
560 for a in assignments:
561 vals = a.split ('=')
562 key = vals[0].strip()
563 val = ('='.join (vals[1:])).strip()
564 if val.endswith ('\"') and val.startswith('"'):
565 val = val[1:-1]
567 values[key] = val
569 paths = values.get("depot-paths")
570 if not paths:
571 paths = values.get("depot-path")
572 if paths:
573 values['depot-paths'] = paths.split(',')
574 return values
576 def gitBranchExists(branch):
577 proc = subprocess.Popen(["git", "rev-parse", branch],
578 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
579 return proc.wait() == 0;
581 _gitConfig = {}
583 def gitConfig(key):
584 if not _gitConfig.has_key(key):
585 cmd = [ "git", "config", key ]
586 s = read_pipe(cmd, ignore_error=True)
587 _gitConfig[key] = s.strip()
588 return _gitConfig[key]
590 def gitConfigBool(key):
591 """Return a bool, using git config --bool. It is True only if the
592 variable is set to true, and False if set to false or not present
593 in the config."""
595 if not _gitConfig.has_key(key):
596 cmd = [ "git", "config", "--bool", key ]
597 s = read_pipe(cmd, ignore_error=True)
598 v = s.strip()
599 _gitConfig[key] = v == "true"
600 return _gitConfig[key]
602 def gitConfigList(key):
603 if not _gitConfig.has_key(key):
604 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
605 _gitConfig[key] = s.strip().split(os.linesep)
606 return _gitConfig[key]
608 def p4BranchesInGit(branchesAreInRemotes=True):
609 """Find all the branches whose names start with "p4/", looking
610 in remotes or heads as specified by the argument. Return
611 a dictionary of { branch: revision } for each one found.
612 The branch names are the short names, without any
613 "p4/" prefix."""
615 branches = {}
617 cmdline = "git rev-parse --symbolic "
618 if branchesAreInRemotes:
619 cmdline += "--remotes"
620 else:
621 cmdline += "--branches"
623 for line in read_pipe_lines(cmdline):
624 line = line.strip()
626 # only import to p4/
627 if not line.startswith('p4/'):
628 continue
629 # special symbolic ref to p4/master
630 if line == "p4/HEAD":
631 continue
633 # strip off p4/ prefix
634 branch = line[len("p4/"):]
636 branches[branch] = parseRevision(line)
638 return branches
640 def branch_exists(branch):
641 """Make sure that the given ref name really exists."""
643 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
644 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
645 out, _ = p.communicate()
646 if p.returncode:
647 return False
648 # expect exactly one line of output: the branch name
649 return out.rstrip() == branch
651 def findUpstreamBranchPoint(head = "HEAD"):
652 branches = p4BranchesInGit()
653 # map from depot-path to branch name
654 branchByDepotPath = {}
655 for branch in branches.keys():
656 tip = branches[branch]
657 log = extractLogMessageFromGitCommit(tip)
658 settings = extractSettingsGitLog(log)
659 if settings.has_key("depot-paths"):
660 paths = ",".join(settings["depot-paths"])
661 branchByDepotPath[paths] = "remotes/p4/" + branch
663 settings = None
664 parent = 0
665 while parent < 65535:
666 commit = head + "~%s" % parent
667 log = extractLogMessageFromGitCommit(commit)
668 settings = extractSettingsGitLog(log)
669 if settings.has_key("depot-paths"):
670 paths = ",".join(settings["depot-paths"])
671 if branchByDepotPath.has_key(paths):
672 return [branchByDepotPath[paths], settings]
674 parent = parent + 1
676 return ["", settings]
678 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
679 if not silent:
680 print ("Creating/updating branch(es) in %s based on origin branch(es)"
681 % localRefPrefix)
683 originPrefix = "origin/p4/"
685 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
686 line = line.strip()
687 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
688 continue
690 headName = line[len(originPrefix):]
691 remoteHead = localRefPrefix + headName
692 originHead = line
694 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
695 if (not original.has_key('depot-paths')
696 or not original.has_key('change')):
697 continue
699 update = False
700 if not gitBranchExists(remoteHead):
701 if verbose:
702 print "creating %s" % remoteHead
703 update = True
704 else:
705 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
706 if settings.has_key('change') > 0:
707 if settings['depot-paths'] == original['depot-paths']:
708 originP4Change = int(original['change'])
709 p4Change = int(settings['change'])
710 if originP4Change > p4Change:
711 print ("%s (%s) is newer than %s (%s). "
712 "Updating p4 branch from origin."
713 % (originHead, originP4Change,
714 remoteHead, p4Change))
715 update = True
716 else:
717 print ("Ignoring: %s was imported from %s while "
718 "%s was imported from %s"
719 % (originHead, ','.join(original['depot-paths']),
720 remoteHead, ','.join(settings['depot-paths'])))
722 if update:
723 system("git update-ref %s %s" % (remoteHead, originHead))
725 def originP4BranchesExist():
726 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
728 def p4ChangesForPaths(depotPaths, changeRange):
729 assert depotPaths
730 cmd = ['changes']
731 for p in depotPaths:
732 cmd += ["%s...%s" % (p, changeRange)]
733 output = p4_read_pipe_lines(cmd)
735 changes = {}
736 for line in output:
737 changeNum = int(line.split(" ")[1])
738 changes[changeNum] = True
740 changelist = changes.keys()
741 changelist.sort()
742 return changelist
744 def p4PathStartsWith(path, prefix):
745 # This method tries to remedy a potential mixed-case issue:
747 # If UserA adds //depot/DirA/file1
748 # and UserB adds //depot/dira/file2
750 # we may or may not have a problem. If you have core.ignorecase=true,
751 # we treat DirA and dira as the same directory
752 if gitConfigBool("core.ignorecase"):
753 return path.lower().startswith(prefix.lower())
754 return path.startswith(prefix)
756 def getClientSpec():
757 """Look at the p4 client spec, create a View() object that contains
758 all the mappings, and return it."""
760 specList = p4CmdList("client -o")
761 if len(specList) != 1:
762 die('Output from "client -o" is %d lines, expecting 1' %
763 len(specList))
765 # dictionary of all client parameters
766 entry = specList[0]
768 # just the keys that start with "View"
769 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
771 # hold this new View
772 view = View()
774 # append the lines, in order, to the view
775 for view_num in range(len(view_keys)):
776 k = "View%d" % view_num
777 if k not in view_keys:
778 die("Expected view key %s missing" % k)
779 view.append(entry[k])
781 return view
783 def getClientRoot():
784 """Grab the client directory."""
786 output = p4CmdList("client -o")
787 if len(output) != 1:
788 die('Output from "client -o" is %d lines, expecting 1' % len(output))
790 entry = output[0]
791 if "Root" not in entry:
792 die('Client has no "Root"')
794 return entry["Root"]
797 # P4 wildcards are not allowed in filenames. P4 complains
798 # if you simply add them, but you can force it with "-f", in
799 # which case it translates them into %xx encoding internally.
801 def wildcard_decode(path):
802 # Search for and fix just these four characters. Do % last so
803 # that fixing it does not inadvertently create new %-escapes.
804 # Cannot have * in a filename in windows; untested as to
805 # what p4 would do in such a case.
806 if not platform.system() == "Windows":
807 path = path.replace("%2A", "*")
808 path = path.replace("%23", "#") \
809 .replace("%40", "@") \
810 .replace("%25", "%")
811 return path
813 def wildcard_encode(path):
814 # do % first to avoid double-encoding the %s introduced here
815 path = path.replace("%", "%25") \
816 .replace("*", "%2A") \
817 .replace("#", "%23") \
818 .replace("@", "%40")
819 return path
821 def wildcard_present(path):
822 m = re.search("[*#@%]", path)
823 return m is not None
825 class Command:
826 def __init__(self):
827 self.usage = "usage: %prog [options]"
828 self.needsGit = True
829 self.verbose = False
831 class P4UserMap:
832 def __init__(self):
833 self.userMapFromPerforceServer = False
834 self.myP4UserId = None
836 def p4UserId(self):
837 if self.myP4UserId:
838 return self.myP4UserId
840 results = p4CmdList("user -o")
841 for r in results:
842 if r.has_key('User'):
843 self.myP4UserId = r['User']
844 return r['User']
845 die("Could not find your p4 user id")
847 def p4UserIsMe(self, p4User):
848 # return True if the given p4 user is actually me
849 me = self.p4UserId()
850 if not p4User or p4User != me:
851 return False
852 else:
853 return True
855 def getUserCacheFilename(self):
856 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
857 return home + "/.gitp4-usercache.txt"
859 def getUserMapFromPerforceServer(self):
860 if self.userMapFromPerforceServer:
861 return
862 self.users = {}
863 self.emails = {}
865 for output in p4CmdList("users"):
866 if not output.has_key("User"):
867 continue
868 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
869 self.emails[output["Email"]] = output["User"]
872 s = ''
873 for (key, val) in self.users.items():
874 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
876 open(self.getUserCacheFilename(), "wb").write(s)
877 self.userMapFromPerforceServer = True
879 def loadUserMapFromCache(self):
880 self.users = {}
881 self.userMapFromPerforceServer = False
882 try:
883 cache = open(self.getUserCacheFilename(), "rb")
884 lines = cache.readlines()
885 cache.close()
886 for line in lines:
887 entry = line.strip().split("\t")
888 self.users[entry[0]] = entry[1]
889 except IOError:
890 self.getUserMapFromPerforceServer()
892 class P4Debug(Command):
893 def __init__(self):
894 Command.__init__(self)
895 self.options = []
896 self.description = "A tool to debug the output of p4 -G."
897 self.needsGit = False
899 def run(self, args):
900 j = 0
901 for output in p4CmdList(args):
902 print 'Element: %d' % j
903 j += 1
904 print output
905 return True
907 class P4RollBack(Command):
908 def __init__(self):
909 Command.__init__(self)
910 self.options = [
911 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
913 self.description = "A tool to debug the multi-branch import. Don't use :)"
914 self.rollbackLocalBranches = False
916 def run(self, args):
917 if len(args) != 1:
918 return False
919 maxChange = int(args[0])
921 if "p4ExitCode" in p4Cmd("changes -m 1"):
922 die("Problems executing p4");
924 if self.rollbackLocalBranches:
925 refPrefix = "refs/heads/"
926 lines = read_pipe_lines("git rev-parse --symbolic --branches")
927 else:
928 refPrefix = "refs/remotes/"
929 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
931 for line in lines:
932 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
933 line = line.strip()
934 ref = refPrefix + line
935 log = extractLogMessageFromGitCommit(ref)
936 settings = extractSettingsGitLog(log)
938 depotPaths = settings['depot-paths']
939 change = settings['change']
941 changed = False
943 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
944 for p in depotPaths]))) == 0:
945 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
946 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
947 continue
949 while change and int(change) > maxChange:
950 changed = True
951 if self.verbose:
952 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
953 system("git update-ref %s \"%s^\"" % (ref, ref))
954 log = extractLogMessageFromGitCommit(ref)
955 settings = extractSettingsGitLog(log)
958 depotPaths = settings['depot-paths']
959 change = settings['change']
961 if changed:
962 print "%s rewound to %s" % (ref, change)
964 return True
966 class P4Submit(Command, P4UserMap):
968 conflict_behavior_choices = ("ask", "skip", "quit")
970 def __init__(self):
971 Command.__init__(self)
972 P4UserMap.__init__(self)
973 self.options = [
974 optparse.make_option("--origin", dest="origin"),
975 optparse.make_option("-M", dest="detectRenames", action="store_true"),
976 # preserve the user, requires relevant p4 permissions
977 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
978 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
979 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
980 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
981 optparse.make_option("--conflict", dest="conflict_behavior",
982 choices=self.conflict_behavior_choices),
983 optparse.make_option("--branch", dest="branch"),
985 self.description = "Submit changes from git to the perforce depot."
986 self.usage += " [name of git branch to submit into perforce depot]"
987 self.origin = ""
988 self.detectRenames = False
989 self.preserveUser = gitConfigBool("git-p4.preserveUser")
990 self.dry_run = False
991 self.prepare_p4_only = False
992 self.conflict_behavior = None
993 self.isWindows = (platform.system() == "Windows")
994 self.exportLabels = False
995 self.p4HasMoveCommand = p4_has_move_command()
996 self.branch = None
998 def check(self):
999 if len(p4CmdList("opened ...")) > 0:
1000 die("You have files opened with perforce! Close them before starting the sync.")
1002 def separate_jobs_from_description(self, message):
1003 """Extract and return a possible Jobs field in the commit
1004 message. It goes into a separate section in the p4 change
1005 specification.
1007 A jobs line starts with "Jobs:" and looks like a new field
1008 in a form. Values are white-space separated on the same
1009 line or on following lines that start with a tab.
1011 This does not parse and extract the full git commit message
1012 like a p4 form. It just sees the Jobs: line as a marker
1013 to pass everything from then on directly into the p4 form,
1014 but outside the description section.
1016 Return a tuple (stripped log message, jobs string)."""
1018 m = re.search(r'^Jobs:', message, re.MULTILINE)
1019 if m is None:
1020 return (message, None)
1022 jobtext = message[m.start():]
1023 stripped_message = message[:m.start()].rstrip()
1024 return (stripped_message, jobtext)
1026 def prepareLogMessage(self, template, message, jobs):
1027 """Edits the template returned from "p4 change -o" to insert
1028 the message in the Description field, and the jobs text in
1029 the Jobs field."""
1030 result = ""
1032 inDescriptionSection = False
1034 for line in template.split("\n"):
1035 if line.startswith("#"):
1036 result += line + "\n"
1037 continue
1039 if inDescriptionSection:
1040 if line.startswith("Files:") or line.startswith("Jobs:"):
1041 inDescriptionSection = False
1042 # insert Jobs section
1043 if jobs:
1044 result += jobs + "\n"
1045 else:
1046 continue
1047 else:
1048 if line.startswith("Description:"):
1049 inDescriptionSection = True
1050 line += "\n"
1051 for messageLine in message.split("\n"):
1052 line += "\t" + messageLine + "\n"
1054 result += line + "\n"
1056 return result
1058 def patchRCSKeywords(self, file, pattern):
1059 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1060 (handle, outFileName) = tempfile.mkstemp(dir='.')
1061 try:
1062 outFile = os.fdopen(handle, "w+")
1063 inFile = open(file, "r")
1064 regexp = re.compile(pattern, re.VERBOSE)
1065 for line in inFile.readlines():
1066 line = regexp.sub(r'$\1$', line)
1067 outFile.write(line)
1068 inFile.close()
1069 outFile.close()
1070 # Forcibly overwrite the original file
1071 os.unlink(file)
1072 shutil.move(outFileName, file)
1073 except:
1074 # cleanup our temporary file
1075 os.unlink(outFileName)
1076 print "Failed to strip RCS keywords in %s" % file
1077 raise
1079 print "Patched up RCS keywords in %s" % file
1081 def p4UserForCommit(self,id):
1082 # Return the tuple (perforce user,git email) for a given git commit id
1083 self.getUserMapFromPerforceServer()
1084 gitEmail = read_pipe(["git", "log", "--max-count=1",
1085 "--format=%ae", id])
1086 gitEmail = gitEmail.strip()
1087 if not self.emails.has_key(gitEmail):
1088 return (None,gitEmail)
1089 else:
1090 return (self.emails[gitEmail],gitEmail)
1092 def checkValidP4Users(self,commits):
1093 # check if any git authors cannot be mapped to p4 users
1094 for id in commits:
1095 (user,email) = self.p4UserForCommit(id)
1096 if not user:
1097 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1098 if gitConfigBool("git-p4.allowMissingP4Users"):
1099 print "%s" % msg
1100 else:
1101 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1103 def lastP4Changelist(self):
1104 # Get back the last changelist number submitted in this client spec. This
1105 # then gets used to patch up the username in the change. If the same
1106 # client spec is being used by multiple processes then this might go
1107 # wrong.
1108 results = p4CmdList("client -o") # find the current client
1109 client = None
1110 for r in results:
1111 if r.has_key('Client'):
1112 client = r['Client']
1113 break
1114 if not client:
1115 die("could not get client spec")
1116 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1117 for r in results:
1118 if r.has_key('change'):
1119 return r['change']
1120 die("Could not get changelist number for last submit - cannot patch up user details")
1122 def modifyChangelistUser(self, changelist, newUser):
1123 # fixup the user field of a changelist after it has been submitted.
1124 changes = p4CmdList("change -o %s" % changelist)
1125 if len(changes) != 1:
1126 die("Bad output from p4 change modifying %s to user %s" %
1127 (changelist, newUser))
1129 c = changes[0]
1130 if c['User'] == newUser: return # nothing to do
1131 c['User'] = newUser
1132 input = marshal.dumps(c)
1134 result = p4CmdList("change -f -i", stdin=input)
1135 for r in result:
1136 if r.has_key('code'):
1137 if r['code'] == 'error':
1138 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1139 if r.has_key('data'):
1140 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1141 return
1142 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1144 def canChangeChangelists(self):
1145 # check to see if we have p4 admin or super-user permissions, either of
1146 # which are required to modify changelists.
1147 results = p4CmdList(["protects", self.depotPath])
1148 for r in results:
1149 if r.has_key('perm'):
1150 if r['perm'] == 'admin':
1151 return 1
1152 if r['perm'] == 'super':
1153 return 1
1154 return 0
1156 def prepareSubmitTemplate(self):
1157 """Run "p4 change -o" to grab a change specification template.
1158 This does not use "p4 -G", as it is nice to keep the submission
1159 template in original order, since a human might edit it.
1161 Remove lines in the Files section that show changes to files
1162 outside the depot path we're committing into."""
1164 template = ""
1165 inFilesSection = False
1166 for line in p4_read_pipe_lines(['change', '-o']):
1167 if line.endswith("\r\n"):
1168 line = line[:-2] + "\n"
1169 if inFilesSection:
1170 if line.startswith("\t"):
1171 # path starts and ends with a tab
1172 path = line[1:]
1173 lastTab = path.rfind("\t")
1174 if lastTab != -1:
1175 path = path[:lastTab]
1176 if not p4PathStartsWith(path, self.depotPath):
1177 continue
1178 else:
1179 inFilesSection = False
1180 else:
1181 if line.startswith("Files:"):
1182 inFilesSection = True
1184 template += line
1186 return template
1188 def edit_template(self, template_file):
1189 """Invoke the editor to let the user change the submission
1190 message. Return true if okay to continue with the submit."""
1192 # if configured to skip the editing part, just submit
1193 if gitConfigBool("git-p4.skipSubmitEdit"):
1194 return True
1196 # look at the modification time, to check later if the user saved
1197 # the file
1198 mtime = os.stat(template_file).st_mtime
1200 # invoke the editor
1201 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1202 editor = os.environ.get("P4EDITOR")
1203 else:
1204 editor = read_pipe("git var GIT_EDITOR").strip()
1205 system(editor + " " + template_file)
1207 # If the file was not saved, prompt to see if this patch should
1208 # be skipped. But skip this verification step if configured so.
1209 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1210 return True
1212 # modification time updated means user saved the file
1213 if os.stat(template_file).st_mtime > mtime:
1214 return True
1216 while True:
1217 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1218 if response == 'y':
1219 return True
1220 if response == 'n':
1221 return False
1223 def applyCommit(self, id):
1224 """Apply one commit, return True if it succeeded."""
1226 print "Applying", read_pipe(["git", "show", "-s",
1227 "--format=format:%h %s", id])
1229 (p4User, gitEmail) = self.p4UserForCommit(id)
1231 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1232 filesToAdd = set()
1233 filesToDelete = set()
1234 editedFiles = set()
1235 pureRenameCopy = set()
1236 filesToChangeExecBit = {}
1238 for line in diff:
1239 diff = parseDiffTreeEntry(line)
1240 modifier = diff['status']
1241 path = diff['src']
1242 if modifier == "M":
1243 p4_edit(path)
1244 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1245 filesToChangeExecBit[path] = diff['dst_mode']
1246 editedFiles.add(path)
1247 elif modifier == "A":
1248 filesToAdd.add(path)
1249 filesToChangeExecBit[path] = diff['dst_mode']
1250 if path in filesToDelete:
1251 filesToDelete.remove(path)
1252 elif modifier == "D":
1253 filesToDelete.add(path)
1254 if path in filesToAdd:
1255 filesToAdd.remove(path)
1256 elif modifier == "C":
1257 src, dest = diff['src'], diff['dst']
1258 p4_integrate(src, dest)
1259 pureRenameCopy.add(dest)
1260 if diff['src_sha1'] != diff['dst_sha1']:
1261 p4_edit(dest)
1262 pureRenameCopy.discard(dest)
1263 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1264 p4_edit(dest)
1265 pureRenameCopy.discard(dest)
1266 filesToChangeExecBit[dest] = diff['dst_mode']
1267 if self.isWindows:
1268 # turn off read-only attribute
1269 os.chmod(dest, stat.S_IWRITE)
1270 os.unlink(dest)
1271 editedFiles.add(dest)
1272 elif modifier == "R":
1273 src, dest = diff['src'], diff['dst']
1274 if self.p4HasMoveCommand:
1275 p4_edit(src) # src must be open before move
1276 p4_move(src, dest) # opens for (move/delete, move/add)
1277 else:
1278 p4_integrate(src, dest)
1279 if diff['src_sha1'] != diff['dst_sha1']:
1280 p4_edit(dest)
1281 else:
1282 pureRenameCopy.add(dest)
1283 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1284 if not self.p4HasMoveCommand:
1285 p4_edit(dest) # with move: already open, writable
1286 filesToChangeExecBit[dest] = diff['dst_mode']
1287 if not self.p4HasMoveCommand:
1288 if self.isWindows:
1289 os.chmod(dest, stat.S_IWRITE)
1290 os.unlink(dest)
1291 filesToDelete.add(src)
1292 editedFiles.add(dest)
1293 else:
1294 die("unknown modifier %s for %s" % (modifier, path))
1296 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1297 patchcmd = diffcmd + " | git apply "
1298 tryPatchCmd = patchcmd + "--check -"
1299 applyPatchCmd = patchcmd + "--check --apply -"
1300 patch_succeeded = True
1302 if os.system(tryPatchCmd) != 0:
1303 fixed_rcs_keywords = False
1304 patch_succeeded = False
1305 print "Unfortunately applying the change failed!"
1307 # Patch failed, maybe it's just RCS keyword woes. Look through
1308 # the patch to see if that's possible.
1309 if gitConfigBool("git-p4.attemptRCSCleanup"):
1310 file = None
1311 pattern = None
1312 kwfiles = {}
1313 for file in editedFiles | filesToDelete:
1314 # did this file's delta contain RCS keywords?
1315 pattern = p4_keywords_regexp_for_file(file)
1317 if pattern:
1318 # this file is a possibility...look for RCS keywords.
1319 regexp = re.compile(pattern, re.VERBOSE)
1320 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1321 if regexp.search(line):
1322 if verbose:
1323 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1324 kwfiles[file] = pattern
1325 break
1327 for file in kwfiles:
1328 if verbose:
1329 print "zapping %s with %s" % (line,pattern)
1330 # File is being deleted, so not open in p4. Must
1331 # disable the read-only bit on windows.
1332 if self.isWindows and file not in editedFiles:
1333 os.chmod(file, stat.S_IWRITE)
1334 self.patchRCSKeywords(file, kwfiles[file])
1335 fixed_rcs_keywords = True
1337 if fixed_rcs_keywords:
1338 print "Retrying the patch with RCS keywords cleaned up"
1339 if os.system(tryPatchCmd) == 0:
1340 patch_succeeded = True
1342 if not patch_succeeded:
1343 for f in editedFiles:
1344 p4_revert(f)
1345 return False
1348 # Apply the patch for real, and do add/delete/+x handling.
1350 system(applyPatchCmd)
1352 for f in filesToAdd:
1353 p4_add(f)
1354 for f in filesToDelete:
1355 p4_revert(f)
1356 p4_delete(f)
1358 # Set/clear executable bits
1359 for f in filesToChangeExecBit.keys():
1360 mode = filesToChangeExecBit[f]
1361 setP4ExecBit(f, mode)
1364 # Build p4 change description, starting with the contents
1365 # of the git commit message.
1367 logMessage = extractLogMessageFromGitCommit(id)
1368 logMessage = logMessage.strip()
1369 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1371 template = self.prepareSubmitTemplate()
1372 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1374 if self.preserveUser:
1375 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1377 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1378 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1379 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1380 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1382 separatorLine = "######## everything below this line is just the diff #######\n"
1384 # diff
1385 if os.environ.has_key("P4DIFF"):
1386 del(os.environ["P4DIFF"])
1387 diff = ""
1388 for editedFile in editedFiles:
1389 diff += p4_read_pipe(['diff', '-du',
1390 wildcard_encode(editedFile)])
1392 # new file diff
1393 newdiff = ""
1394 for newFile in filesToAdd:
1395 newdiff += "==== new file ====\n"
1396 newdiff += "--- /dev/null\n"
1397 newdiff += "+++ %s\n" % newFile
1398 f = open(newFile, "r")
1399 for line in f.readlines():
1400 newdiff += "+" + line
1401 f.close()
1403 # change description file: submitTemplate, separatorLine, diff, newdiff
1404 (handle, fileName) = tempfile.mkstemp()
1405 tmpFile = os.fdopen(handle, "w+")
1406 if self.isWindows:
1407 submitTemplate = submitTemplate.replace("\n", "\r\n")
1408 separatorLine = separatorLine.replace("\n", "\r\n")
1409 newdiff = newdiff.replace("\n", "\r\n")
1410 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1411 tmpFile.close()
1413 if self.prepare_p4_only:
1415 # Leave the p4 tree prepared, and the submit template around
1416 # and let the user decide what to do next
1418 print
1419 print "P4 workspace prepared for submission."
1420 print "To submit or revert, go to client workspace"
1421 print " " + self.clientPath
1422 print
1423 print "To submit, use \"p4 submit\" to write a new description,"
1424 print "or \"p4 submit -i %s\" to use the one prepared by" \
1425 " \"git p4\"." % fileName
1426 print "You can delete the file \"%s\" when finished." % fileName
1428 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1429 print "To preserve change ownership by user %s, you must\n" \
1430 "do \"p4 change -f <change>\" after submitting and\n" \
1431 "edit the User field."
1432 if pureRenameCopy:
1433 print "After submitting, renamed files must be re-synced."
1434 print "Invoke \"p4 sync -f\" on each of these files:"
1435 for f in pureRenameCopy:
1436 print " " + f
1438 print
1439 print "To revert the changes, use \"p4 revert ...\", and delete"
1440 print "the submit template file \"%s\"" % fileName
1441 if filesToAdd:
1442 print "Since the commit adds new files, they must be deleted:"
1443 for f in filesToAdd:
1444 print " " + f
1445 print
1446 return True
1449 # Let the user edit the change description, then submit it.
1451 if self.edit_template(fileName):
1452 # read the edited message and submit
1453 ret = True
1454 tmpFile = open(fileName, "rb")
1455 message = tmpFile.read()
1456 tmpFile.close()
1457 submitTemplate = message[:message.index(separatorLine)]
1458 if self.isWindows:
1459 submitTemplate = submitTemplate.replace("\r\n", "\n")
1460 p4_write_pipe(['submit', '-i'], submitTemplate)
1462 if self.preserveUser:
1463 if p4User:
1464 # Get last changelist number. Cannot easily get it from
1465 # the submit command output as the output is
1466 # unmarshalled.
1467 changelist = self.lastP4Changelist()
1468 self.modifyChangelistUser(changelist, p4User)
1470 # The rename/copy happened by applying a patch that created a
1471 # new file. This leaves it writable, which confuses p4.
1472 for f in pureRenameCopy:
1473 p4_sync(f, "-f")
1475 else:
1476 # skip this patch
1477 ret = False
1478 print "Submission cancelled, undoing p4 changes."
1479 for f in editedFiles:
1480 p4_revert(f)
1481 for f in filesToAdd:
1482 p4_revert(f)
1483 os.remove(f)
1484 for f in filesToDelete:
1485 p4_revert(f)
1487 os.remove(fileName)
1488 return ret
1490 # Export git tags as p4 labels. Create a p4 label and then tag
1491 # with that.
1492 def exportGitTags(self, gitTags):
1493 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1494 if len(validLabelRegexp) == 0:
1495 validLabelRegexp = defaultLabelRegexp
1496 m = re.compile(validLabelRegexp)
1498 for name in gitTags:
1500 if not m.match(name):
1501 if verbose:
1502 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1503 continue
1505 # Get the p4 commit this corresponds to
1506 logMessage = extractLogMessageFromGitCommit(name)
1507 values = extractSettingsGitLog(logMessage)
1509 if not values.has_key('change'):
1510 # a tag pointing to something not sent to p4; ignore
1511 if verbose:
1512 print "git tag %s does not give a p4 commit" % name
1513 continue
1514 else:
1515 changelist = values['change']
1517 # Get the tag details.
1518 inHeader = True
1519 isAnnotated = False
1520 body = []
1521 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1522 l = l.strip()
1523 if inHeader:
1524 if re.match(r'tag\s+', l):
1525 isAnnotated = True
1526 elif re.match(r'\s*$', l):
1527 inHeader = False
1528 continue
1529 else:
1530 body.append(l)
1532 if not isAnnotated:
1533 body = ["lightweight tag imported by git p4\n"]
1535 # Create the label - use the same view as the client spec we are using
1536 clientSpec = getClientSpec()
1538 labelTemplate = "Label: %s\n" % name
1539 labelTemplate += "Description:\n"
1540 for b in body:
1541 labelTemplate += "\t" + b + "\n"
1542 labelTemplate += "View:\n"
1543 for mapping in clientSpec.mappings:
1544 labelTemplate += "\t%s\n" % mapping.depot_side.path
1546 if self.dry_run:
1547 print "Would create p4 label %s for tag" % name
1548 elif self.prepare_p4_only:
1549 print "Not creating p4 label %s for tag due to option" \
1550 " --prepare-p4-only" % name
1551 else:
1552 p4_write_pipe(["label", "-i"], labelTemplate)
1554 # Use the label
1555 p4_system(["tag", "-l", name] +
1556 ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1558 if verbose:
1559 print "created p4 label for tag %s" % name
1561 def run(self, args):
1562 if len(args) == 0:
1563 self.master = currentGitBranch()
1564 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1565 die("Detecting current git branch failed!")
1566 elif len(args) == 1:
1567 self.master = args[0]
1568 if not branchExists(self.master):
1569 die("Branch %s does not exist" % self.master)
1570 else:
1571 return False
1573 allowSubmit = gitConfig("git-p4.allowSubmit")
1574 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1575 die("%s is not in git-p4.allowSubmit" % self.master)
1577 [upstream, settings] = findUpstreamBranchPoint()
1578 self.depotPath = settings['depot-paths'][0]
1579 if len(self.origin) == 0:
1580 self.origin = upstream
1582 if self.preserveUser:
1583 if not self.canChangeChangelists():
1584 die("Cannot preserve user names without p4 super-user or admin permissions")
1586 # if not set from the command line, try the config file
1587 if self.conflict_behavior is None:
1588 val = gitConfig("git-p4.conflict")
1589 if val:
1590 if val not in self.conflict_behavior_choices:
1591 die("Invalid value '%s' for config git-p4.conflict" % val)
1592 else:
1593 val = "ask"
1594 self.conflict_behavior = val
1596 if self.verbose:
1597 print "Origin branch is " + self.origin
1599 if len(self.depotPath) == 0:
1600 print "Internal error: cannot locate perforce depot path from existing branches"
1601 sys.exit(128)
1603 self.useClientSpec = False
1604 if gitConfigBool("git-p4.useclientspec"):
1605 self.useClientSpec = True
1606 if self.useClientSpec:
1607 self.clientSpecDirs = getClientSpec()
1609 if self.useClientSpec:
1610 # all files are relative to the client spec
1611 self.clientPath = getClientRoot()
1612 else:
1613 self.clientPath = p4Where(self.depotPath)
1615 if self.clientPath == "":
1616 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1618 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1619 self.oldWorkingDirectory = os.getcwd()
1621 # ensure the clientPath exists
1622 new_client_dir = False
1623 if not os.path.exists(self.clientPath):
1624 new_client_dir = True
1625 os.makedirs(self.clientPath)
1627 chdir(self.clientPath)
1628 if self.dry_run:
1629 print "Would synchronize p4 checkout in %s" % self.clientPath
1630 else:
1631 print "Synchronizing p4 checkout..."
1632 if new_client_dir:
1633 # old one was destroyed, and maybe nobody told p4
1634 p4_sync("...", "-f")
1635 else:
1636 p4_sync("...")
1637 self.check()
1639 commits = []
1640 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1641 commits.append(line.strip())
1642 commits.reverse()
1644 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1645 self.checkAuthorship = False
1646 else:
1647 self.checkAuthorship = True
1649 if self.preserveUser:
1650 self.checkValidP4Users(commits)
1653 # Build up a set of options to be passed to diff when
1654 # submitting each commit to p4.
1656 if self.detectRenames:
1657 # command-line -M arg
1658 self.diffOpts = "-M"
1659 else:
1660 # If not explicitly set check the config variable
1661 detectRenames = gitConfig("git-p4.detectRenames")
1663 if detectRenames.lower() == "false" or detectRenames == "":
1664 self.diffOpts = ""
1665 elif detectRenames.lower() == "true":
1666 self.diffOpts = "-M"
1667 else:
1668 self.diffOpts = "-M%s" % detectRenames
1670 # no command-line arg for -C or --find-copies-harder, just
1671 # config variables
1672 detectCopies = gitConfig("git-p4.detectCopies")
1673 if detectCopies.lower() == "false" or detectCopies == "":
1674 pass
1675 elif detectCopies.lower() == "true":
1676 self.diffOpts += " -C"
1677 else:
1678 self.diffOpts += " -C%s" % detectCopies
1680 if gitConfigBool("git-p4.detectCopiesHarder"):
1681 self.diffOpts += " --find-copies-harder"
1684 # Apply the commits, one at a time. On failure, ask if should
1685 # continue to try the rest of the patches, or quit.
1687 if self.dry_run:
1688 print "Would apply"
1689 applied = []
1690 last = len(commits) - 1
1691 for i, commit in enumerate(commits):
1692 if self.dry_run:
1693 print " ", read_pipe(["git", "show", "-s",
1694 "--format=format:%h %s", commit])
1695 ok = True
1696 else:
1697 ok = self.applyCommit(commit)
1698 if ok:
1699 applied.append(commit)
1700 else:
1701 if self.prepare_p4_only and i < last:
1702 print "Processing only the first commit due to option" \
1703 " --prepare-p4-only"
1704 break
1705 if i < last:
1706 quit = False
1707 while True:
1708 # prompt for what to do, or use the option/variable
1709 if self.conflict_behavior == "ask":
1710 print "What do you want to do?"
1711 response = raw_input("[s]kip this commit but apply"
1712 " the rest, or [q]uit? ")
1713 if not response:
1714 continue
1715 elif self.conflict_behavior == "skip":
1716 response = "s"
1717 elif self.conflict_behavior == "quit":
1718 response = "q"
1719 else:
1720 die("Unknown conflict_behavior '%s'" %
1721 self.conflict_behavior)
1723 if response[0] == "s":
1724 print "Skipping this commit, but applying the rest"
1725 break
1726 if response[0] == "q":
1727 print "Quitting"
1728 quit = True
1729 break
1730 if quit:
1731 break
1733 chdir(self.oldWorkingDirectory)
1735 if self.dry_run:
1736 pass
1737 elif self.prepare_p4_only:
1738 pass
1739 elif len(commits) == len(applied):
1740 print "All commits applied!"
1742 sync = P4Sync()
1743 if self.branch:
1744 sync.branch = self.branch
1745 sync.run([])
1747 rebase = P4Rebase()
1748 rebase.rebase()
1750 else:
1751 if len(applied) == 0:
1752 print "No commits applied."
1753 else:
1754 print "Applied only the commits marked with '*':"
1755 for c in commits:
1756 if c in applied:
1757 star = "*"
1758 else:
1759 star = " "
1760 print star, read_pipe(["git", "show", "-s",
1761 "--format=format:%h %s", c])
1762 print "You will have to do 'git p4 sync' and rebase."
1764 if gitConfigBool("git-p4.exportLabels"):
1765 self.exportLabels = True
1767 if self.exportLabels:
1768 p4Labels = getP4Labels(self.depotPath)
1769 gitTags = getGitTags()
1771 missingGitTags = gitTags - p4Labels
1772 self.exportGitTags(missingGitTags)
1774 # exit with error unless everything applied perfecly
1775 if len(commits) != len(applied):
1776 sys.exit(1)
1778 return True
1780 class View(object):
1781 """Represent a p4 view ("p4 help views"), and map files in a
1782 repo according to the view."""
1784 class Path(object):
1785 """A depot or client path, possibly containing wildcards.
1786 The only one supported is ... at the end, currently.
1787 Initialize with the full path, with //depot or //client."""
1789 def __init__(self, path, is_depot):
1790 self.path = path
1791 self.is_depot = is_depot
1792 self.find_wildcards()
1793 # remember the prefix bit, useful for relative mappings
1794 m = re.match("(//[^/]+/)", self.path)
1795 if not m:
1796 die("Path %s does not start with //prefix/" % self.path)
1797 prefix = m.group(1)
1798 if not self.is_depot:
1799 # strip //client/ on client paths
1800 self.path = self.path[len(prefix):]
1802 def find_wildcards(self):
1803 """Make sure wildcards are valid, and set up internal
1804 variables."""
1806 self.ends_triple_dot = False
1807 # There are three wildcards allowed in p4 views
1808 # (see "p4 help views"). This code knows how to
1809 # handle "..." (only at the end), but cannot deal with
1810 # "%%n" or "*". Only check the depot_side, as p4 should
1811 # validate that the client_side matches too.
1812 if re.search(r'%%[1-9]', self.path):
1813 die("Can't handle %%n wildcards in view: %s" % self.path)
1814 if self.path.find("*") >= 0:
1815 die("Can't handle * wildcards in view: %s" % self.path)
1816 triple_dot_index = self.path.find("...")
1817 if triple_dot_index >= 0:
1818 if triple_dot_index != len(self.path) - 3:
1819 die("Can handle only single ... wildcard, at end: %s" %
1820 self.path)
1821 self.ends_triple_dot = True
1823 def ensure_compatible(self, other_path):
1824 """Make sure the wildcards agree."""
1825 if self.ends_triple_dot != other_path.ends_triple_dot:
1826 die("Both paths must end with ... if either does;\n" +
1827 "paths: %s %s" % (self.path, other_path.path))
1829 def match_wildcards(self, test_path):
1830 """See if this test_path matches us, and fill in the value
1831 of the wildcards if so. Returns a tuple of
1832 (True|False, wildcards[]). For now, only the ... at end
1833 is supported, so at most one wildcard."""
1834 if self.ends_triple_dot:
1835 dotless = self.path[:-3]
1836 if test_path.startswith(dotless):
1837 wildcard = test_path[len(dotless):]
1838 return (True, [ wildcard ])
1839 else:
1840 if test_path == self.path:
1841 return (True, [])
1842 return (False, [])
1844 def match(self, test_path):
1845 """Just return if it matches; don't bother with the wildcards."""
1846 b, _ = self.match_wildcards(test_path)
1847 return b
1849 def fill_in_wildcards(self, wildcards):
1850 """Return the relative path, with the wildcards filled in
1851 if there are any."""
1852 if self.ends_triple_dot:
1853 return self.path[:-3] + wildcards[0]
1854 else:
1855 return self.path
1857 class Mapping(object):
1858 def __init__(self, depot_side, client_side, overlay, exclude):
1859 # depot_side is without the trailing /... if it had one
1860 self.depot_side = View.Path(depot_side, is_depot=True)
1861 self.client_side = View.Path(client_side, is_depot=False)
1862 self.overlay = overlay # started with "+"
1863 self.exclude = exclude # started with "-"
1864 assert not (self.overlay and self.exclude)
1865 self.depot_side.ensure_compatible(self.client_side)
1867 def __str__(self):
1868 c = " "
1869 if self.overlay:
1870 c = "+"
1871 if self.exclude:
1872 c = "-"
1873 return "View.Mapping: %s%s -> %s" % \
1874 (c, self.depot_side.path, self.client_side.path)
1876 def map_depot_to_client(self, depot_path):
1877 """Calculate the client path if using this mapping on the
1878 given depot path; does not consider the effect of other
1879 mappings in a view. Even excluded mappings are returned."""
1880 matches, wildcards = self.depot_side.match_wildcards(depot_path)
1881 if not matches:
1882 return ""
1883 client_path = self.client_side.fill_in_wildcards(wildcards)
1884 return client_path
1887 # View methods
1889 def __init__(self):
1890 self.mappings = []
1892 def append(self, view_line):
1893 """Parse a view line, splitting it into depot and client
1894 sides. Append to self.mappings, preserving order."""
1896 # Split the view line into exactly two words. P4 enforces
1897 # structure on these lines that simplifies this quite a bit.
1899 # Either or both words may be double-quoted.
1900 # Single quotes do not matter.
1901 # Double-quote marks cannot occur inside the words.
1902 # A + or - prefix is also inside the quotes.
1903 # There are no quotes unless they contain a space.
1904 # The line is already white-space stripped.
1905 # The two words are separated by a single space.
1907 if view_line[0] == '"':
1908 # First word is double quoted. Find its end.
1909 close_quote_index = view_line.find('"', 1)
1910 if close_quote_index <= 0:
1911 die("No first-word closing quote found: %s" % view_line)
1912 depot_side = view_line[1:close_quote_index]
1913 # skip closing quote and space
1914 rhs_index = close_quote_index + 1 + 1
1915 else:
1916 space_index = view_line.find(" ")
1917 if space_index <= 0:
1918 die("No word-splitting space found: %s" % view_line)
1919 depot_side = view_line[0:space_index]
1920 rhs_index = space_index + 1
1922 if view_line[rhs_index] == '"':
1923 # Second word is double quoted. Make sure there is a
1924 # double quote at the end too.
1925 if not view_line.endswith('"'):
1926 die("View line with rhs quote should end with one: %s" %
1927 view_line)
1928 # skip the quotes
1929 client_side = view_line[rhs_index+1:-1]
1930 else:
1931 client_side = view_line[rhs_index:]
1933 # prefix + means overlay on previous mapping
1934 overlay = False
1935 if depot_side.startswith("+"):
1936 overlay = True
1937 depot_side = depot_side[1:]
1939 # prefix - means exclude this path
1940 exclude = False
1941 if depot_side.startswith("-"):
1942 exclude = True
1943 depot_side = depot_side[1:]
1945 m = View.Mapping(depot_side, client_side, overlay, exclude)
1946 self.mappings.append(m)
1948 def map_in_client(self, depot_path):
1949 """Return the relative location in the client where this
1950 depot file should live. Returns "" if the file should
1951 not be mapped in the client."""
1953 paths_filled = []
1954 client_path = ""
1956 # look at later entries first
1957 for m in self.mappings[::-1]:
1959 # see where will this path end up in the client
1960 p = m.map_depot_to_client(depot_path)
1962 if p == "":
1963 # Depot path does not belong in client. Must remember
1964 # this, as previous items should not cause files to
1965 # exist in this path either. Remember that the list is
1966 # being walked from the end, which has higher precedence.
1967 # Overlap mappings do not exclude previous mappings.
1968 if not m.overlay:
1969 paths_filled.append(m.client_side)
1971 else:
1972 # This mapping matched; no need to search any further.
1973 # But, the mapping could be rejected if the client path
1974 # has already been claimed by an earlier mapping (i.e.
1975 # one later in the list, which we are walking backwards).
1976 already_mapped_in_client = False
1977 for f in paths_filled:
1978 # this is View.Path.match
1979 if f.match(p):
1980 already_mapped_in_client = True
1981 break
1982 if not already_mapped_in_client:
1983 # Include this file, unless it is from a line that
1984 # explicitly said to exclude it.
1985 if not m.exclude:
1986 client_path = p
1988 # a match, even if rejected, always stops the search
1989 break
1991 return client_path
1993 class P4Sync(Command, P4UserMap):
1994 delete_actions = ( "delete", "move/delete", "purge" )
1996 def __init__(self):
1997 Command.__init__(self)
1998 P4UserMap.__init__(self)
1999 self.options = [
2000 optparse.make_option("--branch", dest="branch"),
2001 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2002 optparse.make_option("--changesfile", dest="changesFile"),
2003 optparse.make_option("--silent", dest="silent", action="store_true"),
2004 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2005 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2006 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2007 help="Import into refs/heads/ , not refs/remotes"),
2008 optparse.make_option("--max-changes", dest="maxChanges"),
2009 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2010 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2011 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2012 help="Only sync files that are included in the Perforce Client Spec")
2014 self.description = """Imports from Perforce into a git repository.\n
2015 example:
2016 //depot/my/project/ -- to import the current head
2017 //depot/my/project/@all -- to import everything
2018 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2020 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2022 self.usage += " //depot/path[@revRange]"
2023 self.silent = False
2024 self.createdBranches = set()
2025 self.committedChanges = set()
2026 self.branch = ""
2027 self.detectBranches = False
2028 self.detectLabels = False
2029 self.importLabels = False
2030 self.changesFile = ""
2031 self.syncWithOrigin = True
2032 self.importIntoRemotes = True
2033 self.maxChanges = ""
2034 self.keepRepoPath = False
2035 self.depotPaths = None
2036 self.p4BranchesInGit = []
2037 self.cloneExclude = []
2038 self.useClientSpec = False
2039 self.useClientSpec_from_options = False
2040 self.clientSpecDirs = None
2041 self.tempBranches = []
2042 self.tempBranchLocation = "git-p4-tmp"
2044 if gitConfig("git-p4.syncFromOrigin") == "false":
2045 self.syncWithOrigin = False
2047 # Force a checkpoint in fast-import and wait for it to finish
2048 def checkpoint(self):
2049 self.gitStream.write("checkpoint\n\n")
2050 self.gitStream.write("progress checkpoint\n\n")
2051 out = self.gitOutput.readline()
2052 if self.verbose:
2053 print "checkpoint finished: " + out
2055 def extractFilesFromCommit(self, commit):
2056 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2057 for path in self.cloneExclude]
2058 files = []
2059 fnum = 0
2060 while commit.has_key("depotFile%s" % fnum):
2061 path = commit["depotFile%s" % fnum]
2063 if [p for p in self.cloneExclude
2064 if p4PathStartsWith(path, p)]:
2065 found = False
2066 else:
2067 found = [p for p in self.depotPaths
2068 if p4PathStartsWith(path, p)]
2069 if not found:
2070 fnum = fnum + 1
2071 continue
2073 file = {}
2074 file["path"] = path
2075 file["rev"] = commit["rev%s" % fnum]
2076 file["action"] = commit["action%s" % fnum]
2077 file["type"] = commit["type%s" % fnum]
2078 files.append(file)
2079 fnum = fnum + 1
2080 return files
2082 def stripRepoPath(self, path, prefixes):
2083 """When streaming files, this is called to map a p4 depot path
2084 to where it should go in git. The prefixes are either
2085 self.depotPaths, or self.branchPrefixes in the case of
2086 branch detection."""
2088 if self.useClientSpec:
2089 # branch detection moves files up a level (the branch name)
2090 # from what client spec interpretation gives
2091 path = self.clientSpecDirs.map_in_client(path)
2092 if self.detectBranches:
2093 for b in self.knownBranches:
2094 if path.startswith(b + "/"):
2095 path = path[len(b)+1:]
2097 elif self.keepRepoPath:
2098 # Preserve everything in relative path name except leading
2099 # //depot/; just look at first prefix as they all should
2100 # be in the same depot.
2101 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2102 if p4PathStartsWith(path, depot):
2103 path = path[len(depot):]
2105 else:
2106 for p in prefixes:
2107 if p4PathStartsWith(path, p):
2108 path = path[len(p):]
2109 break
2111 path = wildcard_decode(path)
2112 return path
2114 def splitFilesIntoBranches(self, commit):
2115 """Look at each depotFile in the commit to figure out to what
2116 branch it belongs."""
2118 branches = {}
2119 fnum = 0
2120 while commit.has_key("depotFile%s" % fnum):
2121 path = commit["depotFile%s" % fnum]
2122 found = [p for p in self.depotPaths
2123 if p4PathStartsWith(path, p)]
2124 if not found:
2125 fnum = fnum + 1
2126 continue
2128 file = {}
2129 file["path"] = path
2130 file["rev"] = commit["rev%s" % fnum]
2131 file["action"] = commit["action%s" % fnum]
2132 file["type"] = commit["type%s" % fnum]
2133 fnum = fnum + 1
2135 # start with the full relative path where this file would
2136 # go in a p4 client
2137 if self.useClientSpec:
2138 relPath = self.clientSpecDirs.map_in_client(path)
2139 else:
2140 relPath = self.stripRepoPath(path, self.depotPaths)
2142 for branch in self.knownBranches.keys():
2143 # add a trailing slash so that a commit into qt/4.2foo
2144 # doesn't end up in qt/4.2, e.g.
2145 if relPath.startswith(branch + "/"):
2146 if branch not in branches:
2147 branches[branch] = []
2148 branches[branch].append(file)
2149 break
2151 return branches
2153 # output one file from the P4 stream
2154 # - helper for streamP4Files
2156 def streamOneP4File(self, file, contents):
2157 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2158 if verbose:
2159 sys.stderr.write("%s\n" % relPath)
2161 (type_base, type_mods) = split_p4_type(file["type"])
2163 git_mode = "100644"
2164 if "x" in type_mods:
2165 git_mode = "100755"
2166 if type_base == "symlink":
2167 git_mode = "120000"
2168 # p4 print on a symlink contains "target\n"; remove the newline
2169 data = ''.join(contents)
2170 contents = [data[:-1]]
2172 if type_base == "utf16":
2173 # p4 delivers different text in the python output to -G
2174 # than it does when using "print -o", or normal p4 client
2175 # operations. utf16 is converted to ascii or utf8, perhaps.
2176 # But ascii text saved as -t utf16 is completely mangled.
2177 # Invoke print -o to get the real contents.
2179 # On windows, the newlines will always be mangled by print, so put
2180 # them back too. This is not needed to the cygwin windows version,
2181 # just the native "NT" type.
2183 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2184 if p4_version_string().find("/NT") >= 0:
2185 text = text.replace("\r\n", "\n")
2186 contents = [ text ]
2188 if type_base == "apple":
2189 # Apple filetype files will be streamed as a concatenation of
2190 # its appledouble header and the contents. This is useless
2191 # on both macs and non-macs. If using "print -q -o xx", it
2192 # will create "xx" with the data, and "%xx" with the header.
2193 # This is also not very useful.
2195 # Ideally, someday, this script can learn how to generate
2196 # appledouble files directly and import those to git, but
2197 # non-mac machines can never find a use for apple filetype.
2198 print "\nIgnoring apple filetype file %s" % file['depotFile']
2199 return
2201 # Note that we do not try to de-mangle keywords on utf16 files,
2202 # even though in theory somebody may want that.
2203 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2204 if pattern:
2205 regexp = re.compile(pattern, re.VERBOSE)
2206 text = ''.join(contents)
2207 text = regexp.sub(r'$\1$', text)
2208 contents = [ text ]
2210 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2212 # total length...
2213 length = 0
2214 for d in contents:
2215 length = length + len(d)
2217 self.gitStream.write("data %d\n" % length)
2218 for d in contents:
2219 self.gitStream.write(d)
2220 self.gitStream.write("\n")
2222 def streamOneP4Deletion(self, file):
2223 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2224 if verbose:
2225 sys.stderr.write("delete %s\n" % relPath)
2226 self.gitStream.write("D %s\n" % relPath)
2228 # handle another chunk of streaming data
2229 def streamP4FilesCb(self, marshalled):
2231 # catch p4 errors and complain
2232 err = None
2233 if "code" in marshalled:
2234 if marshalled["code"] == "error":
2235 if "data" in marshalled:
2236 err = marshalled["data"].rstrip()
2237 if err:
2238 f = None
2239 if self.stream_have_file_info:
2240 if "depotFile" in self.stream_file:
2241 f = self.stream_file["depotFile"]
2242 # force a failure in fast-import, else an empty
2243 # commit will be made
2244 self.gitStream.write("\n")
2245 self.gitStream.write("die-now\n")
2246 self.gitStream.close()
2247 # ignore errors, but make sure it exits first
2248 self.importProcess.wait()
2249 if f:
2250 die("Error from p4 print for %s: %s" % (f, err))
2251 else:
2252 die("Error from p4 print: %s" % err)
2254 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2255 # start of a new file - output the old one first
2256 self.streamOneP4File(self.stream_file, self.stream_contents)
2257 self.stream_file = {}
2258 self.stream_contents = []
2259 self.stream_have_file_info = False
2261 # pick up the new file information... for the
2262 # 'data' field we need to append to our array
2263 for k in marshalled.keys():
2264 if k == 'data':
2265 self.stream_contents.append(marshalled['data'])
2266 else:
2267 self.stream_file[k] = marshalled[k]
2269 self.stream_have_file_info = True
2271 # Stream directly from "p4 files" into "git fast-import"
2272 def streamP4Files(self, files):
2273 filesForCommit = []
2274 filesToRead = []
2275 filesToDelete = []
2277 for f in files:
2278 # if using a client spec, only add the files that have
2279 # a path in the client
2280 if self.clientSpecDirs:
2281 if self.clientSpecDirs.map_in_client(f['path']) == "":
2282 continue
2284 filesForCommit.append(f)
2285 if f['action'] in self.delete_actions:
2286 filesToDelete.append(f)
2287 else:
2288 filesToRead.append(f)
2290 # deleted files...
2291 for f in filesToDelete:
2292 self.streamOneP4Deletion(f)
2294 if len(filesToRead) > 0:
2295 self.stream_file = {}
2296 self.stream_contents = []
2297 self.stream_have_file_info = False
2299 # curry self argument
2300 def streamP4FilesCbSelf(entry):
2301 self.streamP4FilesCb(entry)
2303 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2305 p4CmdList(["-x", "-", "print"],
2306 stdin=fileArgs,
2307 cb=streamP4FilesCbSelf)
2309 # do the last chunk
2310 if self.stream_file.has_key('depotFile'):
2311 self.streamOneP4File(self.stream_file, self.stream_contents)
2313 def make_email(self, userid):
2314 if userid in self.users:
2315 return self.users[userid]
2316 else:
2317 return "%s <a@b>" % userid
2319 # Stream a p4 tag
2320 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2321 if verbose:
2322 print "writing tag %s for commit %s" % (labelName, commit)
2323 gitStream.write("tag %s\n" % labelName)
2324 gitStream.write("from %s\n" % commit)
2326 if labelDetails.has_key('Owner'):
2327 owner = labelDetails["Owner"]
2328 else:
2329 owner = None
2331 # Try to use the owner of the p4 label, or failing that,
2332 # the current p4 user id.
2333 if owner:
2334 email = self.make_email(owner)
2335 else:
2336 email = self.make_email(self.p4UserId())
2337 tagger = "%s %s %s" % (email, epoch, self.tz)
2339 gitStream.write("tagger %s\n" % tagger)
2341 print "labelDetails=",labelDetails
2342 if labelDetails.has_key('Description'):
2343 description = labelDetails['Description']
2344 else:
2345 description = 'Label from git p4'
2347 gitStream.write("data %d\n" % len(description))
2348 gitStream.write(description)
2349 gitStream.write("\n")
2351 def commit(self, details, files, branch, parent = ""):
2352 epoch = details["time"]
2353 author = details["user"]
2355 if self.verbose:
2356 print "commit into %s" % branch
2358 # start with reading files; if that fails, we should not
2359 # create a commit.
2360 new_files = []
2361 for f in files:
2362 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2363 new_files.append (f)
2364 else:
2365 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2367 self.gitStream.write("commit %s\n" % branch)
2368 # gitStream.write("mark :%s\n" % details["change"])
2369 self.committedChanges.add(int(details["change"]))
2370 committer = ""
2371 if author not in self.users:
2372 self.getUserMapFromPerforceServer()
2373 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2375 self.gitStream.write("committer %s\n" % committer)
2377 self.gitStream.write("data <<EOT\n")
2378 self.gitStream.write(details["desc"])
2379 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2380 (','.join(self.branchPrefixes), details["change"]))
2381 if len(details['options']) > 0:
2382 self.gitStream.write(": options = %s" % details['options'])
2383 self.gitStream.write("]\nEOT\n\n")
2385 if len(parent) > 0:
2386 if self.verbose:
2387 print "parent %s" % parent
2388 self.gitStream.write("from %s\n" % parent)
2390 self.streamP4Files(new_files)
2391 self.gitStream.write("\n")
2393 change = int(details["change"])
2395 if self.labels.has_key(change):
2396 label = self.labels[change]
2397 labelDetails = label[0]
2398 labelRevisions = label[1]
2399 if self.verbose:
2400 print "Change %s is labelled %s" % (change, labelDetails)
2402 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2403 for p in self.branchPrefixes])
2405 if len(files) == len(labelRevisions):
2407 cleanedFiles = {}
2408 for info in files:
2409 if info["action"] in self.delete_actions:
2410 continue
2411 cleanedFiles[info["depotFile"]] = info["rev"]
2413 if cleanedFiles == labelRevisions:
2414 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2416 else:
2417 if not self.silent:
2418 print ("Tag %s does not match with change %s: files do not match."
2419 % (labelDetails["label"], change))
2421 else:
2422 if not self.silent:
2423 print ("Tag %s does not match with change %s: file count is different."
2424 % (labelDetails["label"], change))
2426 # Build a dictionary of changelists and labels, for "detect-labels" option.
2427 def getLabels(self):
2428 self.labels = {}
2430 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2431 if len(l) > 0 and not self.silent:
2432 print "Finding files belonging to labels in %s" % `self.depotPaths`
2434 for output in l:
2435 label = output["label"]
2436 revisions = {}
2437 newestChange = 0
2438 if self.verbose:
2439 print "Querying files for label %s" % label
2440 for file in p4CmdList(["files"] +
2441 ["%s...@%s" % (p, label)
2442 for p in self.depotPaths]):
2443 revisions[file["depotFile"]] = file["rev"]
2444 change = int(file["change"])
2445 if change > newestChange:
2446 newestChange = change
2448 self.labels[newestChange] = [output, revisions]
2450 if self.verbose:
2451 print "Label changes: %s" % self.labels.keys()
2453 # Import p4 labels as git tags. A direct mapping does not
2454 # exist, so assume that if all the files are at the same revision
2455 # then we can use that, or it's something more complicated we should
2456 # just ignore.
2457 def importP4Labels(self, stream, p4Labels):
2458 if verbose:
2459 print "import p4 labels: " + ' '.join(p4Labels)
2461 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2462 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2463 if len(validLabelRegexp) == 0:
2464 validLabelRegexp = defaultLabelRegexp
2465 m = re.compile(validLabelRegexp)
2467 for name in p4Labels:
2468 commitFound = False
2470 if not m.match(name):
2471 if verbose:
2472 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2473 continue
2475 if name in ignoredP4Labels:
2476 continue
2478 labelDetails = p4CmdList(['label', "-o", name])[0]
2480 # get the most recent changelist for each file in this label
2481 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2482 for p in self.depotPaths])
2484 if change.has_key('change'):
2485 # find the corresponding git commit; take the oldest commit
2486 changelist = int(change['change'])
2487 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2488 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2489 if len(gitCommit) == 0:
2490 print "could not find git commit for changelist %d" % changelist
2491 else:
2492 gitCommit = gitCommit.strip()
2493 commitFound = True
2494 # Convert from p4 time format
2495 try:
2496 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2497 except ValueError:
2498 print "Could not convert label time %s" % labelDetails['Update']
2499 tmwhen = 1
2501 when = int(time.mktime(tmwhen))
2502 self.streamTag(stream, name, labelDetails, gitCommit, when)
2503 if verbose:
2504 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2505 else:
2506 if verbose:
2507 print "Label %s has no changelists - possibly deleted?" % name
2509 if not commitFound:
2510 # We can't import this label; don't try again as it will get very
2511 # expensive repeatedly fetching all the files for labels that will
2512 # never be imported. If the label is moved in the future, the
2513 # ignore will need to be removed manually.
2514 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2516 def guessProjectName(self):
2517 for p in self.depotPaths:
2518 if p.endswith("/"):
2519 p = p[:-1]
2520 p = p[p.strip().rfind("/") + 1:]
2521 if not p.endswith("/"):
2522 p += "/"
2523 return p
2525 def getBranchMapping(self):
2526 lostAndFoundBranches = set()
2528 user = gitConfig("git-p4.branchUser")
2529 if len(user) > 0:
2530 command = "branches -u %s" % user
2531 else:
2532 command = "branches"
2534 for info in p4CmdList(command):
2535 details = p4Cmd(["branch", "-o", info["branch"]])
2536 viewIdx = 0
2537 while details.has_key("View%s" % viewIdx):
2538 paths = details["View%s" % viewIdx].split(" ")
2539 viewIdx = viewIdx + 1
2540 # require standard //depot/foo/... //depot/bar/... mapping
2541 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2542 continue
2543 source = paths[0]
2544 destination = paths[1]
2545 ## HACK
2546 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2547 source = source[len(self.depotPaths[0]):-4]
2548 destination = destination[len(self.depotPaths[0]):-4]
2550 if destination in self.knownBranches:
2551 if not self.silent:
2552 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2553 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2554 continue
2556 self.knownBranches[destination] = source
2558 lostAndFoundBranches.discard(destination)
2560 if source not in self.knownBranches:
2561 lostAndFoundBranches.add(source)
2563 # Perforce does not strictly require branches to be defined, so we also
2564 # check git config for a branch list.
2566 # Example of branch definition in git config file:
2567 # [git-p4]
2568 # branchList=main:branchA
2569 # branchList=main:branchB
2570 # branchList=branchA:branchC
2571 configBranches = gitConfigList("git-p4.branchList")
2572 for branch in configBranches:
2573 if branch:
2574 (source, destination) = branch.split(":")
2575 self.knownBranches[destination] = source
2577 lostAndFoundBranches.discard(destination)
2579 if source not in self.knownBranches:
2580 lostAndFoundBranches.add(source)
2583 for branch in lostAndFoundBranches:
2584 self.knownBranches[branch] = branch
2586 def getBranchMappingFromGitBranches(self):
2587 branches = p4BranchesInGit(self.importIntoRemotes)
2588 for branch in branches.keys():
2589 if branch == "master":
2590 branch = "main"
2591 else:
2592 branch = branch[len(self.projectName):]
2593 self.knownBranches[branch] = branch
2595 def updateOptionDict(self, d):
2596 option_keys = {}
2597 if self.keepRepoPath:
2598 option_keys['keepRepoPath'] = 1
2600 d["options"] = ' '.join(sorted(option_keys.keys()))
2602 def readOptions(self, d):
2603 self.keepRepoPath = (d.has_key('options')
2604 and ('keepRepoPath' in d['options']))
2606 def gitRefForBranch(self, branch):
2607 if branch == "main":
2608 return self.refPrefix + "master"
2610 if len(branch) <= 0:
2611 return branch
2613 return self.refPrefix + self.projectName + branch
2615 def gitCommitByP4Change(self, ref, change):
2616 if self.verbose:
2617 print "looking in ref " + ref + " for change %s using bisect..." % change
2619 earliestCommit = ""
2620 latestCommit = parseRevision(ref)
2622 while True:
2623 if self.verbose:
2624 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2625 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2626 if len(next) == 0:
2627 if self.verbose:
2628 print "argh"
2629 return ""
2630 log = extractLogMessageFromGitCommit(next)
2631 settings = extractSettingsGitLog(log)
2632 currentChange = int(settings['change'])
2633 if self.verbose:
2634 print "current change %s" % currentChange
2636 if currentChange == change:
2637 if self.verbose:
2638 print "found %s" % next
2639 return next
2641 if currentChange < change:
2642 earliestCommit = "^%s" % next
2643 else:
2644 latestCommit = "%s" % next
2646 return ""
2648 def importNewBranch(self, branch, maxChange):
2649 # make fast-import flush all changes to disk and update the refs using the checkpoint
2650 # command so that we can try to find the branch parent in the git history
2651 self.gitStream.write("checkpoint\n\n");
2652 self.gitStream.flush();
2653 branchPrefix = self.depotPaths[0] + branch + "/"
2654 range = "@1,%s" % maxChange
2655 #print "prefix" + branchPrefix
2656 changes = p4ChangesForPaths([branchPrefix], range)
2657 if len(changes) <= 0:
2658 return False
2659 firstChange = changes[0]
2660 #print "first change in branch: %s" % firstChange
2661 sourceBranch = self.knownBranches[branch]
2662 sourceDepotPath = self.depotPaths[0] + sourceBranch
2663 sourceRef = self.gitRefForBranch(sourceBranch)
2664 #print "source " + sourceBranch
2666 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2667 #print "branch parent: %s" % branchParentChange
2668 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2669 if len(gitParent) > 0:
2670 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2671 #print "parent git commit: %s" % gitParent
2673 self.importChanges(changes)
2674 return True
2676 def searchParent(self, parent, branch, target):
2677 parentFound = False
2678 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2679 "--no-merges", parent]):
2680 blob = blob.strip()
2681 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2682 parentFound = True
2683 if self.verbose:
2684 print "Found parent of %s in commit %s" % (branch, blob)
2685 break
2686 if parentFound:
2687 return blob
2688 else:
2689 return None
2691 def importChanges(self, changes):
2692 cnt = 1
2693 for change in changes:
2694 description = p4_describe(change)
2695 self.updateOptionDict(description)
2697 if not self.silent:
2698 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2699 sys.stdout.flush()
2700 cnt = cnt + 1
2702 try:
2703 if self.detectBranches:
2704 branches = self.splitFilesIntoBranches(description)
2705 for branch in branches.keys():
2706 ## HACK --hwn
2707 branchPrefix = self.depotPaths[0] + branch + "/"
2708 self.branchPrefixes = [ branchPrefix ]
2710 parent = ""
2712 filesForCommit = branches[branch]
2714 if self.verbose:
2715 print "branch is %s" % branch
2717 self.updatedBranches.add(branch)
2719 if branch not in self.createdBranches:
2720 self.createdBranches.add(branch)
2721 parent = self.knownBranches[branch]
2722 if parent == branch:
2723 parent = ""
2724 else:
2725 fullBranch = self.projectName + branch
2726 if fullBranch not in self.p4BranchesInGit:
2727 if not self.silent:
2728 print("\n Importing new branch %s" % fullBranch);
2729 if self.importNewBranch(branch, change - 1):
2730 parent = ""
2731 self.p4BranchesInGit.append(fullBranch)
2732 if not self.silent:
2733 print("\n Resuming with change %s" % change);
2735 if self.verbose:
2736 print "parent determined through known branches: %s" % parent
2738 branch = self.gitRefForBranch(branch)
2739 parent = self.gitRefForBranch(parent)
2741 if self.verbose:
2742 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2744 if len(parent) == 0 and branch in self.initialParents:
2745 parent = self.initialParents[branch]
2746 del self.initialParents[branch]
2748 blob = None
2749 if len(parent) > 0:
2750 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2751 if self.verbose:
2752 print "Creating temporary branch: " + tempBranch
2753 self.commit(description, filesForCommit, tempBranch)
2754 self.tempBranches.append(tempBranch)
2755 self.checkpoint()
2756 blob = self.searchParent(parent, branch, tempBranch)
2757 if blob:
2758 self.commit(description, filesForCommit, branch, blob)
2759 else:
2760 if self.verbose:
2761 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2762 self.commit(description, filesForCommit, branch, parent)
2763 else:
2764 files = self.extractFilesFromCommit(description)
2765 self.commit(description, files, self.branch,
2766 self.initialParent)
2767 # only needed once, to connect to the previous commit
2768 self.initialParent = ""
2769 except IOError:
2770 print self.gitError.read()
2771 sys.exit(1)
2773 def importHeadRevision(self, revision):
2774 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2776 details = {}
2777 details["user"] = "git perforce import user"
2778 details["desc"] = ("Initial import of %s from the state at revision %s\n"
2779 % (' '.join(self.depotPaths), revision))
2780 details["change"] = revision
2781 newestRevision = 0
2783 fileCnt = 0
2784 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2786 for info in p4CmdList(["files"] + fileArgs):
2788 if 'code' in info and info['code'] == 'error':
2789 sys.stderr.write("p4 returned an error: %s\n"
2790 % info['data'])
2791 if info['data'].find("must refer to client") >= 0:
2792 sys.stderr.write("This particular p4 error is misleading.\n")
2793 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2794 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
2795 sys.exit(1)
2796 if 'p4ExitCode' in info:
2797 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2798 sys.exit(1)
2801 change = int(info["change"])
2802 if change > newestRevision:
2803 newestRevision = change
2805 if info["action"] in self.delete_actions:
2806 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2807 #fileCnt = fileCnt + 1
2808 continue
2810 for prop in ["depotFile", "rev", "action", "type" ]:
2811 details["%s%s" % (prop, fileCnt)] = info[prop]
2813 fileCnt = fileCnt + 1
2815 details["change"] = newestRevision
2817 # Use time from top-most change so that all git p4 clones of
2818 # the same p4 repo have the same commit SHA1s.
2819 res = p4_describe(newestRevision)
2820 details["time"] = res["time"]
2822 self.updateOptionDict(details)
2823 try:
2824 self.commit(details, self.extractFilesFromCommit(details), self.branch)
2825 except IOError:
2826 print "IO error with git fast-import. Is your git version recent enough?"
2827 print self.gitError.read()
2830 def run(self, args):
2831 self.depotPaths = []
2832 self.changeRange = ""
2833 self.previousDepotPaths = []
2834 self.hasOrigin = False
2836 # map from branch depot path to parent branch
2837 self.knownBranches = {}
2838 self.initialParents = {}
2840 if self.importIntoRemotes:
2841 self.refPrefix = "refs/remotes/p4/"
2842 else:
2843 self.refPrefix = "refs/heads/p4/"
2845 if self.syncWithOrigin:
2846 self.hasOrigin = originP4BranchesExist()
2847 if self.hasOrigin:
2848 if not self.silent:
2849 print 'Syncing with origin first, using "git fetch origin"'
2850 system("git fetch origin")
2852 branch_arg_given = bool(self.branch)
2853 if len(self.branch) == 0:
2854 self.branch = self.refPrefix + "master"
2855 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2856 system("git update-ref %s refs/heads/p4" % self.branch)
2857 system("git branch -D p4")
2859 # accept either the command-line option, or the configuration variable
2860 if self.useClientSpec:
2861 # will use this after clone to set the variable
2862 self.useClientSpec_from_options = True
2863 else:
2864 if gitConfigBool("git-p4.useclientspec"):
2865 self.useClientSpec = True
2866 if self.useClientSpec:
2867 self.clientSpecDirs = getClientSpec()
2869 # TODO: should always look at previous commits,
2870 # merge with previous imports, if possible.
2871 if args == []:
2872 if self.hasOrigin:
2873 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2875 # branches holds mapping from branch name to sha1
2876 branches = p4BranchesInGit(self.importIntoRemotes)
2878 # restrict to just this one, disabling detect-branches
2879 if branch_arg_given:
2880 short = self.branch.split("/")[-1]
2881 if short in branches:
2882 self.p4BranchesInGit = [ short ]
2883 else:
2884 self.p4BranchesInGit = branches.keys()
2886 if len(self.p4BranchesInGit) > 1:
2887 if not self.silent:
2888 print "Importing from/into multiple branches"
2889 self.detectBranches = True
2890 for branch in branches.keys():
2891 self.initialParents[self.refPrefix + branch] = \
2892 branches[branch]
2894 if self.verbose:
2895 print "branches: %s" % self.p4BranchesInGit
2897 p4Change = 0
2898 for branch in self.p4BranchesInGit:
2899 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
2901 settings = extractSettingsGitLog(logMsg)
2903 self.readOptions(settings)
2904 if (settings.has_key('depot-paths')
2905 and settings.has_key ('change')):
2906 change = int(settings['change']) + 1
2907 p4Change = max(p4Change, change)
2909 depotPaths = sorted(settings['depot-paths'])
2910 if self.previousDepotPaths == []:
2911 self.previousDepotPaths = depotPaths
2912 else:
2913 paths = []
2914 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2915 prev_list = prev.split("/")
2916 cur_list = cur.split("/")
2917 for i in range(0, min(len(cur_list), len(prev_list))):
2918 if cur_list[i] <> prev_list[i]:
2919 i = i - 1
2920 break
2922 paths.append ("/".join(cur_list[:i + 1]))
2924 self.previousDepotPaths = paths
2926 if p4Change > 0:
2927 self.depotPaths = sorted(self.previousDepotPaths)
2928 self.changeRange = "@%s,#head" % p4Change
2929 if not self.silent and not self.detectBranches:
2930 print "Performing incremental import into %s git branch" % self.branch
2932 # accept multiple ref name abbreviations:
2933 # refs/foo/bar/branch -> use it exactly
2934 # p4/branch -> prepend refs/remotes/ or refs/heads/
2935 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2936 if not self.branch.startswith("refs/"):
2937 if self.importIntoRemotes:
2938 prepend = "refs/remotes/"
2939 else:
2940 prepend = "refs/heads/"
2941 if not self.branch.startswith("p4/"):
2942 prepend += "p4/"
2943 self.branch = prepend + self.branch
2945 if len(args) == 0 and self.depotPaths:
2946 if not self.silent:
2947 print "Depot paths: %s" % ' '.join(self.depotPaths)
2948 else:
2949 if self.depotPaths and self.depotPaths != args:
2950 print ("previous import used depot path %s and now %s was specified. "
2951 "This doesn't work!" % (' '.join (self.depotPaths),
2952 ' '.join (args)))
2953 sys.exit(1)
2955 self.depotPaths = sorted(args)
2957 revision = ""
2958 self.users = {}
2960 # Make sure no revision specifiers are used when --changesfile
2961 # is specified.
2962 bad_changesfile = False
2963 if len(self.changesFile) > 0:
2964 for p in self.depotPaths:
2965 if p.find("@") >= 0 or p.find("#") >= 0:
2966 bad_changesfile = True
2967 break
2968 if bad_changesfile:
2969 die("Option --changesfile is incompatible with revision specifiers")
2971 newPaths = []
2972 for p in self.depotPaths:
2973 if p.find("@") != -1:
2974 atIdx = p.index("@")
2975 self.changeRange = p[atIdx:]
2976 if self.changeRange == "@all":
2977 self.changeRange = ""
2978 elif ',' not in self.changeRange:
2979 revision = self.changeRange
2980 self.changeRange = ""
2981 p = p[:atIdx]
2982 elif p.find("#") != -1:
2983 hashIdx = p.index("#")
2984 revision = p[hashIdx:]
2985 p = p[:hashIdx]
2986 elif self.previousDepotPaths == []:
2987 # pay attention to changesfile, if given, else import
2988 # the entire p4 tree at the head revision
2989 if len(self.changesFile) == 0:
2990 revision = "#head"
2992 p = re.sub ("\.\.\.$", "", p)
2993 if not p.endswith("/"):
2994 p += "/"
2996 newPaths.append(p)
2998 self.depotPaths = newPaths
3000 # --detect-branches may change this for each branch
3001 self.branchPrefixes = self.depotPaths
3003 self.loadUserMapFromCache()
3004 self.labels = {}
3005 if self.detectLabels:
3006 self.getLabels();
3008 if self.detectBranches:
3009 ## FIXME - what's a P4 projectName ?
3010 self.projectName = self.guessProjectName()
3012 if self.hasOrigin:
3013 self.getBranchMappingFromGitBranches()
3014 else:
3015 self.getBranchMapping()
3016 if self.verbose:
3017 print "p4-git branches: %s" % self.p4BranchesInGit
3018 print "initial parents: %s" % self.initialParents
3019 for b in self.p4BranchesInGit:
3020 if b != "master":
3022 ## FIXME
3023 b = b[len(self.projectName):]
3024 self.createdBranches.add(b)
3026 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3028 self.importProcess = subprocess.Popen(["git", "fast-import"],
3029 stdin=subprocess.PIPE,
3030 stdout=subprocess.PIPE,
3031 stderr=subprocess.PIPE);
3032 self.gitOutput = self.importProcess.stdout
3033 self.gitStream = self.importProcess.stdin
3034 self.gitError = self.importProcess.stderr
3036 if revision:
3037 self.importHeadRevision(revision)
3038 else:
3039 changes = []
3041 if len(self.changesFile) > 0:
3042 output = open(self.changesFile).readlines()
3043 changeSet = set()
3044 for line in output:
3045 changeSet.add(int(line))
3047 for change in changeSet:
3048 changes.append(change)
3050 changes.sort()
3051 else:
3052 # catch "git p4 sync" with no new branches, in a repo that
3053 # does not have any existing p4 branches
3054 if len(args) == 0:
3055 if not self.p4BranchesInGit:
3056 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3058 # The default branch is master, unless --branch is used to
3059 # specify something else. Make sure it exists, or complain
3060 # nicely about how to use --branch.
3061 if not self.detectBranches:
3062 if not branch_exists(self.branch):
3063 if branch_arg_given:
3064 die("Error: branch %s does not exist." % self.branch)
3065 else:
3066 die("Error: no branch %s; perhaps specify one with --branch." %
3067 self.branch)
3069 if self.verbose:
3070 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3071 self.changeRange)
3072 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
3074 if len(self.maxChanges) > 0:
3075 changes = changes[:min(int(self.maxChanges), len(changes))]
3077 if len(changes) == 0:
3078 if not self.silent:
3079 print "No changes to import!"
3080 else:
3081 if not self.silent and not self.detectBranches:
3082 print "Import destination: %s" % self.branch
3084 self.updatedBranches = set()
3086 if not self.detectBranches:
3087 if args:
3088 # start a new branch
3089 self.initialParent = ""
3090 else:
3091 # build on a previous revision
3092 self.initialParent = parseRevision(self.branch)
3094 self.importChanges(changes)
3096 if not self.silent:
3097 print ""
3098 if len(self.updatedBranches) > 0:
3099 sys.stdout.write("Updated branches: ")
3100 for b in self.updatedBranches:
3101 sys.stdout.write("%s " % b)
3102 sys.stdout.write("\n")
3104 if gitConfigBool("git-p4.importLabels"):
3105 self.importLabels = True
3107 if self.importLabels:
3108 p4Labels = getP4Labels(self.depotPaths)
3109 gitTags = getGitTags()
3111 missingP4Labels = p4Labels - gitTags
3112 self.importP4Labels(self.gitStream, missingP4Labels)
3114 self.gitStream.close()
3115 if self.importProcess.wait() != 0:
3116 die("fast-import failed: %s" % self.gitError.read())
3117 self.gitOutput.close()
3118 self.gitError.close()
3120 # Cleanup temporary branches created during import
3121 if self.tempBranches != []:
3122 for branch in self.tempBranches:
3123 read_pipe("git update-ref -d %s" % branch)
3124 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3126 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3127 # a convenient shortcut refname "p4".
3128 if self.importIntoRemotes:
3129 head_ref = self.refPrefix + "HEAD"
3130 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3131 system(["git", "symbolic-ref", head_ref, self.branch])
3133 return True
3135 class P4Rebase(Command):
3136 def __init__(self):
3137 Command.__init__(self)
3138 self.options = [
3139 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3141 self.importLabels = False
3142 self.description = ("Fetches the latest revision from perforce and "
3143 + "rebases the current work (branch) against it")
3145 def run(self, args):
3146 sync = P4Sync()
3147 sync.importLabels = self.importLabels
3148 sync.run([])
3150 return self.rebase()
3152 def rebase(self):
3153 if os.system("git update-index --refresh") != 0:
3154 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3155 if len(read_pipe("git diff-index HEAD --")) > 0:
3156 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
3158 [upstream, settings] = findUpstreamBranchPoint()
3159 if len(upstream) == 0:
3160 die("Cannot find upstream branchpoint for rebase")
3162 # the branchpoint may be p4/foo~3, so strip off the parent
3163 upstream = re.sub("~[0-9]+$", "", upstream)
3165 print "Rebasing the current branch onto %s" % upstream
3166 oldHead = read_pipe("git rev-parse HEAD").strip()
3167 system("git rebase %s" % upstream)
3168 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3169 return True
3171 class P4Clone(P4Sync):
3172 def __init__(self):
3173 P4Sync.__init__(self)
3174 self.description = "Creates a new git repository and imports from Perforce into it"
3175 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3176 self.options += [
3177 optparse.make_option("--destination", dest="cloneDestination",
3178 action='store', default=None,
3179 help="where to leave result of the clone"),
3180 optparse.make_option("-/", dest="cloneExclude",
3181 action="append", type="string",
3182 help="exclude depot path"),
3183 optparse.make_option("--bare", dest="cloneBare",
3184 action="store_true", default=False),
3186 self.cloneDestination = None
3187 self.needsGit = False
3188 self.cloneBare = False
3190 # This is required for the "append" cloneExclude action
3191 def ensure_value(self, attr, value):
3192 if not hasattr(self, attr) or getattr(self, attr) is None:
3193 setattr(self, attr, value)
3194 return getattr(self, attr)
3196 def defaultDestination(self, args):
3197 ## TODO: use common prefix of args?
3198 depotPath = args[0]
3199 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3200 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3201 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3202 depotDir = re.sub(r"/$", "", depotDir)
3203 return os.path.split(depotDir)[1]
3205 def run(self, args):
3206 if len(args) < 1:
3207 return False
3209 if self.keepRepoPath and not self.cloneDestination:
3210 sys.stderr.write("Must specify destination for --keep-path\n")
3211 sys.exit(1)
3213 depotPaths = args
3215 if not self.cloneDestination and len(depotPaths) > 1:
3216 self.cloneDestination = depotPaths[-1]
3217 depotPaths = depotPaths[:-1]
3219 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3220 for p in depotPaths:
3221 if not p.startswith("//"):
3222 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3223 return False
3225 if not self.cloneDestination:
3226 self.cloneDestination = self.defaultDestination(args)
3228 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3230 if not os.path.exists(self.cloneDestination):
3231 os.makedirs(self.cloneDestination)
3232 chdir(self.cloneDestination)
3234 init_cmd = [ "git", "init" ]
3235 if self.cloneBare:
3236 init_cmd.append("--bare")
3237 retcode = subprocess.call(init_cmd)
3238 if retcode:
3239 raise CalledProcessError(retcode, init_cmd)
3241 if not P4Sync.run(self, depotPaths):
3242 return False
3244 # create a master branch and check out a work tree
3245 if gitBranchExists(self.branch):
3246 system([ "git", "branch", "master", self.branch ])
3247 if not self.cloneBare:
3248 system([ "git", "checkout", "-f" ])
3249 else:
3250 print 'Not checking out any branch, use ' \
3251 '"git checkout -q -b master <branch>"'
3253 # auto-set this variable if invoked with --use-client-spec
3254 if self.useClientSpec_from_options:
3255 system("git config --bool git-p4.useclientspec true")
3257 return True
3259 class P4Branches(Command):
3260 def __init__(self):
3261 Command.__init__(self)
3262 self.options = [ ]
3263 self.description = ("Shows the git branches that hold imports and their "
3264 + "corresponding perforce depot paths")
3265 self.verbose = False
3267 def run(self, args):
3268 if originP4BranchesExist():
3269 createOrUpdateBranchesFromOrigin()
3271 cmdline = "git rev-parse --symbolic "
3272 cmdline += " --remotes"
3274 for line in read_pipe_lines(cmdline):
3275 line = line.strip()
3277 if not line.startswith('p4/') or line == "p4/HEAD":
3278 continue
3279 branch = line
3281 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3282 settings = extractSettingsGitLog(log)
3284 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3285 return True
3287 class HelpFormatter(optparse.IndentedHelpFormatter):
3288 def __init__(self):
3289 optparse.IndentedHelpFormatter.__init__(self)
3291 def format_description(self, description):
3292 if description:
3293 return description + "\n"
3294 else:
3295 return ""
3297 def printUsage(commands):
3298 print "usage: %s <command> [options]" % sys.argv[0]
3299 print ""
3300 print "valid commands: %s" % ", ".join(commands)
3301 print ""
3302 print "Try %s <command> --help for command specific help." % sys.argv[0]
3303 print ""
3305 commands = {
3306 "debug" : P4Debug,
3307 "submit" : P4Submit,
3308 "commit" : P4Submit,
3309 "sync" : P4Sync,
3310 "rebase" : P4Rebase,
3311 "clone" : P4Clone,
3312 "rollback" : P4RollBack,
3313 "branches" : P4Branches
3317 def main():
3318 if len(sys.argv[1:]) == 0:
3319 printUsage(commands.keys())
3320 sys.exit(2)
3322 cmdName = sys.argv[1]
3323 try:
3324 klass = commands[cmdName]
3325 cmd = klass()
3326 except KeyError:
3327 print "unknown command %s" % cmdName
3328 print ""
3329 printUsage(commands.keys())
3330 sys.exit(2)
3332 options = cmd.options
3333 cmd.gitdir = os.environ.get("GIT_DIR", None)
3335 args = sys.argv[2:]
3337 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3338 if cmd.needsGit:
3339 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3341 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3342 options,
3343 description = cmd.description,
3344 formatter = HelpFormatter())
3346 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3347 global verbose
3348 verbose = cmd.verbose
3349 if cmd.needsGit:
3350 if cmd.gitdir == None:
3351 cmd.gitdir = os.path.abspath(".git")
3352 if not isValidGitDir(cmd.gitdir):
3353 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3354 if os.path.exists(cmd.gitdir):
3355 cdup = read_pipe("git rev-parse --show-cdup").strip()
3356 if len(cdup) > 0:
3357 chdir(cdup);
3359 if not isValidGitDir(cmd.gitdir):
3360 if isValidGitDir(cmd.gitdir + "/.git"):
3361 cmd.gitdir += "/.git"
3362 else:
3363 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3365 os.environ["GIT_DIR"] = cmd.gitdir
3367 if not cmd.run(args):
3368 parser.print_help()
3369 sys.exit(2)
3372 if __name__ == '__main__':
3373 main()