Interested in helping open source friends on HP-UX?
[git/mjg.git] / git-p4.py
blob549022e97c83e4a2f1898dc005331e5bca3f0ded
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
26 try:
27 from subprocess import CalledProcessError
28 except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
41 verbose = False
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
46 def p4_build_cmd(cmd):
47 """Build a suitable p4 command line.
49 This consolidates building and returning a p4 command line into one
50 location. It means that hooking into the environment, or other configuration
51 can be done more easily.
52 """
53 real_cmd = ["p4"]
55 user = gitConfig("git-p4.user")
56 if len(user) > 0:
57 real_cmd += ["-u",user]
59 password = gitConfig("git-p4.password")
60 if len(password) > 0:
61 real_cmd += ["-P", password]
63 port = gitConfig("git-p4.port")
64 if len(port) > 0:
65 real_cmd += ["-p", port]
67 host = gitConfig("git-p4.host")
68 if len(host) > 0:
69 real_cmd += ["-H", host]
71 client = gitConfig("git-p4.client")
72 if len(client) > 0:
73 real_cmd += ["-c", client]
76 if isinstance(cmd,basestring):
77 real_cmd = ' '.join(real_cmd) + ' ' + cmd
78 else:
79 real_cmd += cmd
80 return real_cmd
82 def chdir(path, is_client_path=False):
83 """Do chdir to the given path, and set the PWD environment
84 variable for use by P4. It does not look at getcwd() output.
85 Since we're not using the shell, it is necessary to set the
86 PWD environment variable explicitly.
88 Normally, expand the path to force it to be absolute. This
89 addresses the use of relative path names inside P4 settings,
90 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
91 as given; it looks for .p4config using PWD.
93 If is_client_path, the path was handed to us directly by p4,
94 and may be a symbolic link. Do not call os.getcwd() in this
95 case, because it will cause p4 to think that PWD is not inside
96 the client path.
97 """
99 os.chdir(path)
100 if not is_client_path:
101 path = os.getcwd()
102 os.environ['PWD'] = path
104 def die(msg):
105 if verbose:
106 raise Exception(msg)
107 else:
108 sys.stderr.write(msg + "\n")
109 sys.exit(1)
111 def write_pipe(c, stdin):
112 if verbose:
113 sys.stderr.write('Writing pipe: %s\n' % str(c))
115 expand = isinstance(c,basestring)
116 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
117 pipe = p.stdin
118 val = pipe.write(stdin)
119 pipe.close()
120 if p.wait():
121 die('Command failed: %s' % str(c))
123 return val
125 def p4_write_pipe(c, stdin):
126 real_cmd = p4_build_cmd(c)
127 return write_pipe(real_cmd, stdin)
129 def read_pipe(c, ignore_error=False):
130 if verbose:
131 sys.stderr.write('Reading pipe: %s\n' % str(c))
133 expand = isinstance(c,basestring)
134 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
135 pipe = p.stdout
136 val = pipe.read()
137 if p.wait() and not ignore_error:
138 die('Command failed: %s' % str(c))
140 return val
142 def p4_read_pipe(c, ignore_error=False):
143 real_cmd = p4_build_cmd(c)
144 return read_pipe(real_cmd, ignore_error)
146 def read_pipe_lines(c):
147 if verbose:
148 sys.stderr.write('Reading pipe: %s\n' % str(c))
150 expand = isinstance(c, basestring)
151 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
152 pipe = p.stdout
153 val = pipe.readlines()
154 if pipe.close() or p.wait():
155 die('Command failed: %s' % str(c))
157 return val
159 def p4_read_pipe_lines(c):
160 """Specifically invoke p4 on the command supplied. """
161 real_cmd = p4_build_cmd(c)
162 return read_pipe_lines(real_cmd)
164 def p4_has_command(cmd):
165 """Ask p4 for help on this command. If it returns an error, the
166 command does not exist in this version of p4."""
167 real_cmd = p4_build_cmd(["help", cmd])
168 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
169 stderr=subprocess.PIPE)
170 p.communicate()
171 return p.returncode == 0
173 def p4_has_move_command():
174 """See if the move command exists, that it supports -k, and that
175 it has not been administratively disabled. The arguments
176 must be correct, but the filenames do not have to exist. Use
177 ones with wildcards so even if they exist, it will fail."""
179 if not p4_has_command("move"):
180 return False
181 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
182 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183 (out, err) = p.communicate()
184 # return code will be 1 in either case
185 if err.find("Invalid option") >= 0:
186 return False
187 if err.find("disabled") >= 0:
188 return False
189 # assume it failed because @... was invalid changelist
190 return True
192 def system(cmd):
193 expand = isinstance(cmd,basestring)
194 if verbose:
195 sys.stderr.write("executing %s\n" % str(cmd))
196 retcode = subprocess.call(cmd, shell=expand)
197 if retcode:
198 raise CalledProcessError(retcode, cmd)
200 def p4_system(cmd):
201 """Specifically invoke p4 as the system command. """
202 real_cmd = p4_build_cmd(cmd)
203 expand = isinstance(real_cmd, basestring)
204 retcode = subprocess.call(real_cmd, shell=expand)
205 if retcode:
206 raise CalledProcessError(retcode, real_cmd)
208 _p4_version_string = None
209 def p4_version_string():
210 """Read the version string, showing just the last line, which
211 hopefully is the interesting version bit.
213 $ p4 -V
214 Perforce - The Fast Software Configuration Management System.
215 Copyright 1995-2011 Perforce Software. All rights reserved.
216 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
218 global _p4_version_string
219 if not _p4_version_string:
220 a = p4_read_pipe_lines(["-V"])
221 _p4_version_string = a[-1].rstrip()
222 return _p4_version_string
224 def p4_integrate(src, dest):
225 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
227 def p4_sync(f, *options):
228 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
230 def p4_add(f):
231 # forcibly add file names with wildcards
232 if wildcard_present(f):
233 p4_system(["add", "-f", f])
234 else:
235 p4_system(["add", f])
237 def p4_delete(f):
238 p4_system(["delete", wildcard_encode(f)])
240 def p4_edit(f):
241 p4_system(["edit", wildcard_encode(f)])
243 def p4_revert(f):
244 p4_system(["revert", wildcard_encode(f)])
246 def p4_reopen(type, f):
247 p4_system(["reopen", "-t", type, wildcard_encode(f)])
249 def p4_move(src, dest):
250 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
252 def p4_describe(change):
253 """Make sure it returns a valid result by checking for
254 the presence of field "time". Return a dict of the
255 results."""
257 ds = p4CmdList(["describe", "-s", str(change)])
258 if len(ds) != 1:
259 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
261 d = ds[0]
263 if "p4ExitCode" in d:
264 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
265 str(d)))
266 if "code" in d:
267 if d["code"] == "error":
268 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
270 if "time" not in d:
271 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
273 return d
276 # Canonicalize the p4 type and return a tuple of the
277 # base type, plus any modifiers. See "p4 help filetypes"
278 # for a list and explanation.
280 def split_p4_type(p4type):
282 p4_filetypes_historical = {
283 "ctempobj": "binary+Sw",
284 "ctext": "text+C",
285 "cxtext": "text+Cx",
286 "ktext": "text+k",
287 "kxtext": "text+kx",
288 "ltext": "text+F",
289 "tempobj": "binary+FSw",
290 "ubinary": "binary+F",
291 "uresource": "resource+F",
292 "uxbinary": "binary+Fx",
293 "xbinary": "binary+x",
294 "xltext": "text+Fx",
295 "xtempobj": "binary+Swx",
296 "xtext": "text+x",
297 "xunicode": "unicode+x",
298 "xutf16": "utf16+x",
300 if p4type in p4_filetypes_historical:
301 p4type = p4_filetypes_historical[p4type]
302 mods = ""
303 s = p4type.split("+")
304 base = s[0]
305 mods = ""
306 if len(s) > 1:
307 mods = s[1]
308 return (base, mods)
311 # return the raw p4 type of a file (text, text+ko, etc)
313 def p4_type(f):
314 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
315 return results[0]['headType']
318 # Given a type base and modifier, return a regexp matching
319 # the keywords that can be expanded in the file
321 def p4_keywords_regexp_for_type(base, type_mods):
322 if base in ("text", "unicode", "binary"):
323 kwords = None
324 if "ko" in type_mods:
325 kwords = 'Id|Header'
326 elif "k" in type_mods:
327 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
328 else:
329 return None
330 pattern = r"""
331 \$ # Starts with a dollar, followed by...
332 (%s) # one of the keywords, followed by...
333 (:[^$\n]+)? # possibly an old expansion, followed by...
334 \$ # another dollar
335 """ % kwords
336 return pattern
337 else:
338 return None
341 # Given a file, return a regexp matching the possible
342 # RCS keywords that will be expanded, or None for files
343 # with kw expansion turned off.
345 def p4_keywords_regexp_for_file(file):
346 if not os.path.exists(file):
347 return None
348 else:
349 (type_base, type_mods) = split_p4_type(p4_type(file))
350 return p4_keywords_regexp_for_type(type_base, type_mods)
352 def setP4ExecBit(file, mode):
353 # Reopens an already open file and changes the execute bit to match
354 # the execute bit setting in the passed in mode.
356 p4Type = "+x"
358 if not isModeExec(mode):
359 p4Type = getP4OpenedType(file)
360 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
361 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
362 if p4Type[-1] == "+":
363 p4Type = p4Type[0:-1]
365 p4_reopen(p4Type, file)
367 def getP4OpenedType(file):
368 # Returns the perforce file type for the given file.
370 result = p4_read_pipe(["opened", wildcard_encode(file)])
371 match = re.match(".*\((.+)\)\r?$", result)
372 if match:
373 return match.group(1)
374 else:
375 die("Could not determine file type for %s (result: '%s')" % (file, result))
377 # Return the set of all p4 labels
378 def getP4Labels(depotPaths):
379 labels = set()
380 if isinstance(depotPaths,basestring):
381 depotPaths = [depotPaths]
383 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
384 label = l['label']
385 labels.add(label)
387 return labels
389 # Return the set of all git tags
390 def getGitTags():
391 gitTags = set()
392 for line in read_pipe_lines(["git", "tag"]):
393 tag = line.strip()
394 gitTags.add(tag)
395 return gitTags
397 def diffTreePattern():
398 # This is a simple generator for the diff tree regex pattern. This could be
399 # a class variable if this and parseDiffTreeEntry were a part of a class.
400 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
401 while True:
402 yield pattern
404 def parseDiffTreeEntry(entry):
405 """Parses a single diff tree entry into its component elements.
407 See git-diff-tree(1) manpage for details about the format of the diff
408 output. This method returns a dictionary with the following elements:
410 src_mode - The mode of the source file
411 dst_mode - The mode of the destination file
412 src_sha1 - The sha1 for the source file
413 dst_sha1 - The sha1 fr the destination file
414 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
415 status_score - The score for the status (applicable for 'C' and 'R'
416 statuses). This is None if there is no score.
417 src - The path for the source file.
418 dst - The path for the destination file. This is only present for
419 copy or renames. If it is not present, this is None.
421 If the pattern is not matched, None is returned."""
423 match = diffTreePattern().next().match(entry)
424 if match:
425 return {
426 'src_mode': match.group(1),
427 'dst_mode': match.group(2),
428 'src_sha1': match.group(3),
429 'dst_sha1': match.group(4),
430 'status': match.group(5),
431 'status_score': match.group(6),
432 'src': match.group(7),
433 'dst': match.group(10)
435 return None
437 def isModeExec(mode):
438 # Returns True if the given git mode represents an executable file,
439 # otherwise False.
440 return mode[-3:] == "755"
442 def isModeExecChanged(src_mode, dst_mode):
443 return isModeExec(src_mode) != isModeExec(dst_mode)
445 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
447 if isinstance(cmd,basestring):
448 cmd = "-G " + cmd
449 expand = True
450 else:
451 cmd = ["-G"] + cmd
452 expand = False
454 cmd = p4_build_cmd(cmd)
455 if verbose:
456 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
458 # Use a temporary file to avoid deadlocks without
459 # subprocess.communicate(), which would put another copy
460 # of stdout into memory.
461 stdin_file = None
462 if stdin is not None:
463 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
464 if isinstance(stdin,basestring):
465 stdin_file.write(stdin)
466 else:
467 for i in stdin:
468 stdin_file.write(i + '\n')
469 stdin_file.flush()
470 stdin_file.seek(0)
472 p4 = subprocess.Popen(cmd,
473 shell=expand,
474 stdin=stdin_file,
475 stdout=subprocess.PIPE)
477 result = []
478 try:
479 while True:
480 entry = marshal.load(p4.stdout)
481 if cb is not None:
482 cb(entry)
483 else:
484 result.append(entry)
485 except EOFError:
486 pass
487 exitCode = p4.wait()
488 if exitCode != 0:
489 entry = {}
490 entry["p4ExitCode"] = exitCode
491 result.append(entry)
493 return result
495 def p4Cmd(cmd):
496 list = p4CmdList(cmd)
497 result = {}
498 for entry in list:
499 result.update(entry)
500 return result;
502 def p4Where(depotPath):
503 if not depotPath.endswith("/"):
504 depotPath += "/"
505 depotPath = depotPath + "..."
506 outputList = p4CmdList(["where", depotPath])
507 output = None
508 for entry in outputList:
509 if "depotFile" in entry:
510 if entry["depotFile"] == depotPath:
511 output = entry
512 break
513 elif "data" in entry:
514 data = entry.get("data")
515 space = data.find(" ")
516 if data[:space] == depotPath:
517 output = entry
518 break
519 if output == None:
520 return ""
521 if output["code"] == "error":
522 return ""
523 clientPath = ""
524 if "path" in output:
525 clientPath = output.get("path")
526 elif "data" in output:
527 data = output.get("data")
528 lastSpace = data.rfind(" ")
529 clientPath = data[lastSpace + 1:]
531 if clientPath.endswith("..."):
532 clientPath = clientPath[:-3]
533 return clientPath
535 def currentGitBranch():
536 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
538 def isValidGitDir(path):
539 if (os.path.exists(path + "/HEAD")
540 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
541 return True;
542 return False
544 def parseRevision(ref):
545 return read_pipe("git rev-parse %s" % ref).strip()
547 def branchExists(ref):
548 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
549 ignore_error=True)
550 return len(rev) > 0
552 def extractLogMessageFromGitCommit(commit):
553 logMessage = ""
555 ## fixme: title is first line of commit, not 1st paragraph.
556 foundTitle = False
557 for log in read_pipe_lines("git cat-file commit %s" % commit):
558 if not foundTitle:
559 if len(log) == 1:
560 foundTitle = True
561 continue
563 logMessage += log
564 return logMessage
566 def extractSettingsGitLog(log):
567 values = {}
568 for line in log.split("\n"):
569 line = line.strip()
570 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
571 if not m:
572 continue
574 assignments = m.group(1).split (':')
575 for a in assignments:
576 vals = a.split ('=')
577 key = vals[0].strip()
578 val = ('='.join (vals[1:])).strip()
579 if val.endswith ('\"') and val.startswith('"'):
580 val = val[1:-1]
582 values[key] = val
584 paths = values.get("depot-paths")
585 if not paths:
586 paths = values.get("depot-path")
587 if paths:
588 values['depot-paths'] = paths.split(',')
589 return values
591 def gitBranchExists(branch):
592 proc = subprocess.Popen(["git", "rev-parse", branch],
593 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
594 return proc.wait() == 0;
596 _gitConfig = {}
598 def gitConfig(key):
599 if not _gitConfig.has_key(key):
600 cmd = [ "git", "config", key ]
601 s = read_pipe(cmd, ignore_error=True)
602 _gitConfig[key] = s.strip()
603 return _gitConfig[key]
605 def gitConfigBool(key):
606 """Return a bool, using git config --bool. It is True only if the
607 variable is set to true, and False if set to false or not present
608 in the config."""
610 if not _gitConfig.has_key(key):
611 cmd = [ "git", "config", "--bool", key ]
612 s = read_pipe(cmd, ignore_error=True)
613 v = s.strip()
614 _gitConfig[key] = v == "true"
615 return _gitConfig[key]
617 def gitConfigList(key):
618 if not _gitConfig.has_key(key):
619 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
620 _gitConfig[key] = s.strip().split(os.linesep)
621 return _gitConfig[key]
623 def p4BranchesInGit(branchesAreInRemotes=True):
624 """Find all the branches whose names start with "p4/", looking
625 in remotes or heads as specified by the argument. Return
626 a dictionary of { branch: revision } for each one found.
627 The branch names are the short names, without any
628 "p4/" prefix."""
630 branches = {}
632 cmdline = "git rev-parse --symbolic "
633 if branchesAreInRemotes:
634 cmdline += "--remotes"
635 else:
636 cmdline += "--branches"
638 for line in read_pipe_lines(cmdline):
639 line = line.strip()
641 # only import to p4/
642 if not line.startswith('p4/'):
643 continue
644 # special symbolic ref to p4/master
645 if line == "p4/HEAD":
646 continue
648 # strip off p4/ prefix
649 branch = line[len("p4/"):]
651 branches[branch] = parseRevision(line)
653 return branches
655 def branch_exists(branch):
656 """Make sure that the given ref name really exists."""
658 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
659 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
660 out, _ = p.communicate()
661 if p.returncode:
662 return False
663 # expect exactly one line of output: the branch name
664 return out.rstrip() == branch
666 def findUpstreamBranchPoint(head = "HEAD"):
667 branches = p4BranchesInGit()
668 # map from depot-path to branch name
669 branchByDepotPath = {}
670 for branch in branches.keys():
671 tip = branches[branch]
672 log = extractLogMessageFromGitCommit(tip)
673 settings = extractSettingsGitLog(log)
674 if settings.has_key("depot-paths"):
675 paths = ",".join(settings["depot-paths"])
676 branchByDepotPath[paths] = "remotes/p4/" + branch
678 settings = None
679 parent = 0
680 while parent < 65535:
681 commit = head + "~%s" % parent
682 log = extractLogMessageFromGitCommit(commit)
683 settings = extractSettingsGitLog(log)
684 if settings.has_key("depot-paths"):
685 paths = ",".join(settings["depot-paths"])
686 if branchByDepotPath.has_key(paths):
687 return [branchByDepotPath[paths], settings]
689 parent = parent + 1
691 return ["", settings]
693 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
694 if not silent:
695 print ("Creating/updating branch(es) in %s based on origin branch(es)"
696 % localRefPrefix)
698 originPrefix = "origin/p4/"
700 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
701 line = line.strip()
702 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
703 continue
705 headName = line[len(originPrefix):]
706 remoteHead = localRefPrefix + headName
707 originHead = line
709 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
710 if (not original.has_key('depot-paths')
711 or not original.has_key('change')):
712 continue
714 update = False
715 if not gitBranchExists(remoteHead):
716 if verbose:
717 print "creating %s" % remoteHead
718 update = True
719 else:
720 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
721 if settings.has_key('change') > 0:
722 if settings['depot-paths'] == original['depot-paths']:
723 originP4Change = int(original['change'])
724 p4Change = int(settings['change'])
725 if originP4Change > p4Change:
726 print ("%s (%s) is newer than %s (%s). "
727 "Updating p4 branch from origin."
728 % (originHead, originP4Change,
729 remoteHead, p4Change))
730 update = True
731 else:
732 print ("Ignoring: %s was imported from %s while "
733 "%s was imported from %s"
734 % (originHead, ','.join(original['depot-paths']),
735 remoteHead, ','.join(settings['depot-paths'])))
737 if update:
738 system("git update-ref %s %s" % (remoteHead, originHead))
740 def originP4BranchesExist():
741 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
743 def p4ChangesForPaths(depotPaths, changeRange):
744 assert depotPaths
745 cmd = ['changes']
746 for p in depotPaths:
747 cmd += ["%s...%s" % (p, changeRange)]
748 output = p4_read_pipe_lines(cmd)
750 changes = {}
751 for line in output:
752 changeNum = int(line.split(" ")[1])
753 changes[changeNum] = True
755 changelist = changes.keys()
756 changelist.sort()
757 return changelist
759 def p4PathStartsWith(path, prefix):
760 # This method tries to remedy a potential mixed-case issue:
762 # If UserA adds //depot/DirA/file1
763 # and UserB adds //depot/dira/file2
765 # we may or may not have a problem. If you have core.ignorecase=true,
766 # we treat DirA and dira as the same directory
767 if gitConfigBool("core.ignorecase"):
768 return path.lower().startswith(prefix.lower())
769 return path.startswith(prefix)
771 def getClientSpec():
772 """Look at the p4 client spec, create a View() object that contains
773 all the mappings, and return it."""
775 specList = p4CmdList("client -o")
776 if len(specList) != 1:
777 die('Output from "client -o" is %d lines, expecting 1' %
778 len(specList))
780 # dictionary of all client parameters
781 entry = specList[0]
783 # the //client/ name
784 client_name = entry["Client"]
786 # just the keys that start with "View"
787 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
789 # hold this new View
790 view = View(client_name)
792 # append the lines, in order, to the view
793 for view_num in range(len(view_keys)):
794 k = "View%d" % view_num
795 if k not in view_keys:
796 die("Expected view key %s missing" % k)
797 view.append(entry[k])
799 return view
801 def getClientRoot():
802 """Grab the client directory."""
804 output = p4CmdList("client -o")
805 if len(output) != 1:
806 die('Output from "client -o" is %d lines, expecting 1' % len(output))
808 entry = output[0]
809 if "Root" not in entry:
810 die('Client has no "Root"')
812 return entry["Root"]
815 # P4 wildcards are not allowed in filenames. P4 complains
816 # if you simply add them, but you can force it with "-f", in
817 # which case it translates them into %xx encoding internally.
819 def wildcard_decode(path):
820 # Search for and fix just these four characters. Do % last so
821 # that fixing it does not inadvertently create new %-escapes.
822 # Cannot have * in a filename in windows; untested as to
823 # what p4 would do in such a case.
824 if not platform.system() == "Windows":
825 path = path.replace("%2A", "*")
826 path = path.replace("%23", "#") \
827 .replace("%40", "@") \
828 .replace("%25", "%")
829 return path
831 def wildcard_encode(path):
832 # do % first to avoid double-encoding the %s introduced here
833 path = path.replace("%", "%25") \
834 .replace("*", "%2A") \
835 .replace("#", "%23") \
836 .replace("@", "%40")
837 return path
839 def wildcard_present(path):
840 m = re.search("[*#@%]", path)
841 return m is not None
843 class Command:
844 def __init__(self):
845 self.usage = "usage: %prog [options]"
846 self.needsGit = True
847 self.verbose = False
849 class P4UserMap:
850 def __init__(self):
851 self.userMapFromPerforceServer = False
852 self.myP4UserId = None
854 def p4UserId(self):
855 if self.myP4UserId:
856 return self.myP4UserId
858 results = p4CmdList("user -o")
859 for r in results:
860 if r.has_key('User'):
861 self.myP4UserId = r['User']
862 return r['User']
863 die("Could not find your p4 user id")
865 def p4UserIsMe(self, p4User):
866 # return True if the given p4 user is actually me
867 me = self.p4UserId()
868 if not p4User or p4User != me:
869 return False
870 else:
871 return True
873 def getUserCacheFilename(self):
874 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
875 return home + "/.gitp4-usercache.txt"
877 def getUserMapFromPerforceServer(self):
878 if self.userMapFromPerforceServer:
879 return
880 self.users = {}
881 self.emails = {}
883 for output in p4CmdList("users"):
884 if not output.has_key("User"):
885 continue
886 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
887 self.emails[output["Email"]] = output["User"]
890 s = ''
891 for (key, val) in self.users.items():
892 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
894 open(self.getUserCacheFilename(), "wb").write(s)
895 self.userMapFromPerforceServer = True
897 def loadUserMapFromCache(self):
898 self.users = {}
899 self.userMapFromPerforceServer = False
900 try:
901 cache = open(self.getUserCacheFilename(), "rb")
902 lines = cache.readlines()
903 cache.close()
904 for line in lines:
905 entry = line.strip().split("\t")
906 self.users[entry[0]] = entry[1]
907 except IOError:
908 self.getUserMapFromPerforceServer()
910 class P4Debug(Command):
911 def __init__(self):
912 Command.__init__(self)
913 self.options = []
914 self.description = "A tool to debug the output of p4 -G."
915 self.needsGit = False
917 def run(self, args):
918 j = 0
919 for output in p4CmdList(args):
920 print 'Element: %d' % j
921 j += 1
922 print output
923 return True
925 class P4RollBack(Command):
926 def __init__(self):
927 Command.__init__(self)
928 self.options = [
929 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
931 self.description = "A tool to debug the multi-branch import. Don't use :)"
932 self.rollbackLocalBranches = False
934 def run(self, args):
935 if len(args) != 1:
936 return False
937 maxChange = int(args[0])
939 if "p4ExitCode" in p4Cmd("changes -m 1"):
940 die("Problems executing p4");
942 if self.rollbackLocalBranches:
943 refPrefix = "refs/heads/"
944 lines = read_pipe_lines("git rev-parse --symbolic --branches")
945 else:
946 refPrefix = "refs/remotes/"
947 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
949 for line in lines:
950 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
951 line = line.strip()
952 ref = refPrefix + line
953 log = extractLogMessageFromGitCommit(ref)
954 settings = extractSettingsGitLog(log)
956 depotPaths = settings['depot-paths']
957 change = settings['change']
959 changed = False
961 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
962 for p in depotPaths]))) == 0:
963 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
964 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
965 continue
967 while change and int(change) > maxChange:
968 changed = True
969 if self.verbose:
970 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
971 system("git update-ref %s \"%s^\"" % (ref, ref))
972 log = extractLogMessageFromGitCommit(ref)
973 settings = extractSettingsGitLog(log)
976 depotPaths = settings['depot-paths']
977 change = settings['change']
979 if changed:
980 print "%s rewound to %s" % (ref, change)
982 return True
984 class P4Submit(Command, P4UserMap):
986 conflict_behavior_choices = ("ask", "skip", "quit")
988 def __init__(self):
989 Command.__init__(self)
990 P4UserMap.__init__(self)
991 self.options = [
992 optparse.make_option("--origin", dest="origin"),
993 optparse.make_option("-M", dest="detectRenames", action="store_true"),
994 # preserve the user, requires relevant p4 permissions
995 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
996 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
997 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
998 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
999 optparse.make_option("--conflict", dest="conflict_behavior",
1000 choices=self.conflict_behavior_choices),
1001 optparse.make_option("--branch", dest="branch"),
1003 self.description = "Submit changes from git to the perforce depot."
1004 self.usage += " [name of git branch to submit into perforce depot]"
1005 self.origin = ""
1006 self.detectRenames = False
1007 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1008 self.dry_run = False
1009 self.prepare_p4_only = False
1010 self.conflict_behavior = None
1011 self.isWindows = (platform.system() == "Windows")
1012 self.exportLabels = False
1013 self.p4HasMoveCommand = p4_has_move_command()
1014 self.branch = None
1016 def check(self):
1017 if len(p4CmdList("opened ...")) > 0:
1018 die("You have files opened with perforce! Close them before starting the sync.")
1020 def separate_jobs_from_description(self, message):
1021 """Extract and return a possible Jobs field in the commit
1022 message. It goes into a separate section in the p4 change
1023 specification.
1025 A jobs line starts with "Jobs:" and looks like a new field
1026 in a form. Values are white-space separated on the same
1027 line or on following lines that start with a tab.
1029 This does not parse and extract the full git commit message
1030 like a p4 form. It just sees the Jobs: line as a marker
1031 to pass everything from then on directly into the p4 form,
1032 but outside the description section.
1034 Return a tuple (stripped log message, jobs string)."""
1036 m = re.search(r'^Jobs:', message, re.MULTILINE)
1037 if m is None:
1038 return (message, None)
1040 jobtext = message[m.start():]
1041 stripped_message = message[:m.start()].rstrip()
1042 return (stripped_message, jobtext)
1044 def prepareLogMessage(self, template, message, jobs):
1045 """Edits the template returned from "p4 change -o" to insert
1046 the message in the Description field, and the jobs text in
1047 the Jobs field."""
1048 result = ""
1050 inDescriptionSection = False
1052 for line in template.split("\n"):
1053 if line.startswith("#"):
1054 result += line + "\n"
1055 continue
1057 if inDescriptionSection:
1058 if line.startswith("Files:") or line.startswith("Jobs:"):
1059 inDescriptionSection = False
1060 # insert Jobs section
1061 if jobs:
1062 result += jobs + "\n"
1063 else:
1064 continue
1065 else:
1066 if line.startswith("Description:"):
1067 inDescriptionSection = True
1068 line += "\n"
1069 for messageLine in message.split("\n"):
1070 line += "\t" + messageLine + "\n"
1072 result += line + "\n"
1074 return result
1076 def patchRCSKeywords(self, file, pattern):
1077 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1078 (handle, outFileName) = tempfile.mkstemp(dir='.')
1079 try:
1080 outFile = os.fdopen(handle, "w+")
1081 inFile = open(file, "r")
1082 regexp = re.compile(pattern, re.VERBOSE)
1083 for line in inFile.readlines():
1084 line = regexp.sub(r'$\1$', line)
1085 outFile.write(line)
1086 inFile.close()
1087 outFile.close()
1088 # Forcibly overwrite the original file
1089 os.unlink(file)
1090 shutil.move(outFileName, file)
1091 except:
1092 # cleanup our temporary file
1093 os.unlink(outFileName)
1094 print "Failed to strip RCS keywords in %s" % file
1095 raise
1097 print "Patched up RCS keywords in %s" % file
1099 def p4UserForCommit(self,id):
1100 # Return the tuple (perforce user,git email) for a given git commit id
1101 self.getUserMapFromPerforceServer()
1102 gitEmail = read_pipe(["git", "log", "--max-count=1",
1103 "--format=%ae", id])
1104 gitEmail = gitEmail.strip()
1105 if not self.emails.has_key(gitEmail):
1106 return (None,gitEmail)
1107 else:
1108 return (self.emails[gitEmail],gitEmail)
1110 def checkValidP4Users(self,commits):
1111 # check if any git authors cannot be mapped to p4 users
1112 for id in commits:
1113 (user,email) = self.p4UserForCommit(id)
1114 if not user:
1115 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1116 if gitConfigBool("git-p4.allowMissingP4Users"):
1117 print "%s" % msg
1118 else:
1119 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1121 def lastP4Changelist(self):
1122 # Get back the last changelist number submitted in this client spec. This
1123 # then gets used to patch up the username in the change. If the same
1124 # client spec is being used by multiple processes then this might go
1125 # wrong.
1126 results = p4CmdList("client -o") # find the current client
1127 client = None
1128 for r in results:
1129 if r.has_key('Client'):
1130 client = r['Client']
1131 break
1132 if not client:
1133 die("could not get client spec")
1134 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1135 for r in results:
1136 if r.has_key('change'):
1137 return r['change']
1138 die("Could not get changelist number for last submit - cannot patch up user details")
1140 def modifyChangelistUser(self, changelist, newUser):
1141 # fixup the user field of a changelist after it has been submitted.
1142 changes = p4CmdList("change -o %s" % changelist)
1143 if len(changes) != 1:
1144 die("Bad output from p4 change modifying %s to user %s" %
1145 (changelist, newUser))
1147 c = changes[0]
1148 if c['User'] == newUser: return # nothing to do
1149 c['User'] = newUser
1150 input = marshal.dumps(c)
1152 result = p4CmdList("change -f -i", stdin=input)
1153 for r in result:
1154 if r.has_key('code'):
1155 if r['code'] == 'error':
1156 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1157 if r.has_key('data'):
1158 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1159 return
1160 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1162 def canChangeChangelists(self):
1163 # check to see if we have p4 admin or super-user permissions, either of
1164 # which are required to modify changelists.
1165 results = p4CmdList(["protects", self.depotPath])
1166 for r in results:
1167 if r.has_key('perm'):
1168 if r['perm'] == 'admin':
1169 return 1
1170 if r['perm'] == 'super':
1171 return 1
1172 return 0
1174 def prepareSubmitTemplate(self):
1175 """Run "p4 change -o" to grab a change specification template.
1176 This does not use "p4 -G", as it is nice to keep the submission
1177 template in original order, since a human might edit it.
1179 Remove lines in the Files section that show changes to files
1180 outside the depot path we're committing into."""
1182 template = ""
1183 inFilesSection = False
1184 for line in p4_read_pipe_lines(['change', '-o']):
1185 if line.endswith("\r\n"):
1186 line = line[:-2] + "\n"
1187 if inFilesSection:
1188 if line.startswith("\t"):
1189 # path starts and ends with a tab
1190 path = line[1:]
1191 lastTab = path.rfind("\t")
1192 if lastTab != -1:
1193 path = path[:lastTab]
1194 if not p4PathStartsWith(path, self.depotPath):
1195 continue
1196 else:
1197 inFilesSection = False
1198 else:
1199 if line.startswith("Files:"):
1200 inFilesSection = True
1202 template += line
1204 return template
1206 def edit_template(self, template_file):
1207 """Invoke the editor to let the user change the submission
1208 message. Return true if okay to continue with the submit."""
1210 # if configured to skip the editing part, just submit
1211 if gitConfigBool("git-p4.skipSubmitEdit"):
1212 return True
1214 # look at the modification time, to check later if the user saved
1215 # the file
1216 mtime = os.stat(template_file).st_mtime
1218 # invoke the editor
1219 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1220 editor = os.environ.get("P4EDITOR")
1221 else:
1222 editor = read_pipe("git var GIT_EDITOR").strip()
1223 system([editor, template_file])
1225 # If the file was not saved, prompt to see if this patch should
1226 # be skipped. But skip this verification step if configured so.
1227 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1228 return True
1230 # modification time updated means user saved the file
1231 if os.stat(template_file).st_mtime > mtime:
1232 return True
1234 while True:
1235 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1236 if response == 'y':
1237 return True
1238 if response == 'n':
1239 return False
1241 def get_diff_description(self, editedFiles, filesToAdd):
1242 # diff
1243 if os.environ.has_key("P4DIFF"):
1244 del(os.environ["P4DIFF"])
1245 diff = ""
1246 for editedFile in editedFiles:
1247 diff += p4_read_pipe(['diff', '-du',
1248 wildcard_encode(editedFile)])
1250 # new file diff
1251 newdiff = ""
1252 for newFile in filesToAdd:
1253 newdiff += "==== new file ====\n"
1254 newdiff += "--- /dev/null\n"
1255 newdiff += "+++ %s\n" % newFile
1256 f = open(newFile, "r")
1257 for line in f.readlines():
1258 newdiff += "+" + line
1259 f.close()
1261 return (diff + newdiff).replace('\r\n', '\n')
1263 def applyCommit(self, id):
1264 """Apply one commit, return True if it succeeded."""
1266 print "Applying", read_pipe(["git", "show", "-s",
1267 "--format=format:%h %s", id])
1269 (p4User, gitEmail) = self.p4UserForCommit(id)
1271 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1272 filesToAdd = set()
1273 filesToDelete = set()
1274 editedFiles = set()
1275 pureRenameCopy = set()
1276 filesToChangeExecBit = {}
1278 for line in diff:
1279 diff = parseDiffTreeEntry(line)
1280 modifier = diff['status']
1281 path = diff['src']
1282 if modifier == "M":
1283 p4_edit(path)
1284 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1285 filesToChangeExecBit[path] = diff['dst_mode']
1286 editedFiles.add(path)
1287 elif modifier == "A":
1288 filesToAdd.add(path)
1289 filesToChangeExecBit[path] = diff['dst_mode']
1290 if path in filesToDelete:
1291 filesToDelete.remove(path)
1292 elif modifier == "D":
1293 filesToDelete.add(path)
1294 if path in filesToAdd:
1295 filesToAdd.remove(path)
1296 elif modifier == "C":
1297 src, dest = diff['src'], diff['dst']
1298 p4_integrate(src, dest)
1299 pureRenameCopy.add(dest)
1300 if diff['src_sha1'] != diff['dst_sha1']:
1301 p4_edit(dest)
1302 pureRenameCopy.discard(dest)
1303 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1304 p4_edit(dest)
1305 pureRenameCopy.discard(dest)
1306 filesToChangeExecBit[dest] = diff['dst_mode']
1307 if self.isWindows:
1308 # turn off read-only attribute
1309 os.chmod(dest, stat.S_IWRITE)
1310 os.unlink(dest)
1311 editedFiles.add(dest)
1312 elif modifier == "R":
1313 src, dest = diff['src'], diff['dst']
1314 if self.p4HasMoveCommand:
1315 p4_edit(src) # src must be open before move
1316 p4_move(src, dest) # opens for (move/delete, move/add)
1317 else:
1318 p4_integrate(src, dest)
1319 if diff['src_sha1'] != diff['dst_sha1']:
1320 p4_edit(dest)
1321 else:
1322 pureRenameCopy.add(dest)
1323 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1324 if not self.p4HasMoveCommand:
1325 p4_edit(dest) # with move: already open, writable
1326 filesToChangeExecBit[dest] = diff['dst_mode']
1327 if not self.p4HasMoveCommand:
1328 if self.isWindows:
1329 os.chmod(dest, stat.S_IWRITE)
1330 os.unlink(dest)
1331 filesToDelete.add(src)
1332 editedFiles.add(dest)
1333 else:
1334 die("unknown modifier %s for %s" % (modifier, path))
1336 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1337 patchcmd = diffcmd + " | git apply "
1338 tryPatchCmd = patchcmd + "--check -"
1339 applyPatchCmd = patchcmd + "--check --apply -"
1340 patch_succeeded = True
1342 if os.system(tryPatchCmd) != 0:
1343 fixed_rcs_keywords = False
1344 patch_succeeded = False
1345 print "Unfortunately applying the change failed!"
1347 # Patch failed, maybe it's just RCS keyword woes. Look through
1348 # the patch to see if that's possible.
1349 if gitConfigBool("git-p4.attemptRCSCleanup"):
1350 file = None
1351 pattern = None
1352 kwfiles = {}
1353 for file in editedFiles | filesToDelete:
1354 # did this file's delta contain RCS keywords?
1355 pattern = p4_keywords_regexp_for_file(file)
1357 if pattern:
1358 # this file is a possibility...look for RCS keywords.
1359 regexp = re.compile(pattern, re.VERBOSE)
1360 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1361 if regexp.search(line):
1362 if verbose:
1363 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1364 kwfiles[file] = pattern
1365 break
1367 for file in kwfiles:
1368 if verbose:
1369 print "zapping %s with %s" % (line,pattern)
1370 # File is being deleted, so not open in p4. Must
1371 # disable the read-only bit on windows.
1372 if self.isWindows and file not in editedFiles:
1373 os.chmod(file, stat.S_IWRITE)
1374 self.patchRCSKeywords(file, kwfiles[file])
1375 fixed_rcs_keywords = True
1377 if fixed_rcs_keywords:
1378 print "Retrying the patch with RCS keywords cleaned up"
1379 if os.system(tryPatchCmd) == 0:
1380 patch_succeeded = True
1382 if not patch_succeeded:
1383 for f in editedFiles:
1384 p4_revert(f)
1385 return False
1388 # Apply the patch for real, and do add/delete/+x handling.
1390 system(applyPatchCmd)
1392 for f in filesToAdd:
1393 p4_add(f)
1394 for f in filesToDelete:
1395 p4_revert(f)
1396 p4_delete(f)
1398 # Set/clear executable bits
1399 for f in filesToChangeExecBit.keys():
1400 mode = filesToChangeExecBit[f]
1401 setP4ExecBit(f, mode)
1404 # Build p4 change description, starting with the contents
1405 # of the git commit message.
1407 logMessage = extractLogMessageFromGitCommit(id)
1408 logMessage = logMessage.strip()
1409 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1411 template = self.prepareSubmitTemplate()
1412 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1414 if self.preserveUser:
1415 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1417 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1418 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1419 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1420 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1422 separatorLine = "######## everything below this line is just the diff #######\n"
1423 if not self.prepare_p4_only:
1424 submitTemplate += separatorLine
1425 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1427 (handle, fileName) = tempfile.mkstemp()
1428 tmpFile = os.fdopen(handle, "w+b")
1429 if self.isWindows:
1430 submitTemplate = submitTemplate.replace("\n", "\r\n")
1431 tmpFile.write(submitTemplate)
1432 tmpFile.close()
1434 if self.prepare_p4_only:
1436 # Leave the p4 tree prepared, and the submit template around
1437 # and let the user decide what to do next
1439 print
1440 print "P4 workspace prepared for submission."
1441 print "To submit or revert, go to client workspace"
1442 print " " + self.clientPath
1443 print
1444 print "To submit, use \"p4 submit\" to write a new description,"
1445 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1446 " \"git p4\"." % fileName
1447 print "You can delete the file \"%s\" when finished." % fileName
1449 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1450 print "To preserve change ownership by user %s, you must\n" \
1451 "do \"p4 change -f <change>\" after submitting and\n" \
1452 "edit the User field."
1453 if pureRenameCopy:
1454 print "After submitting, renamed files must be re-synced."
1455 print "Invoke \"p4 sync -f\" on each of these files:"
1456 for f in pureRenameCopy:
1457 print " " + f
1459 print
1460 print "To revert the changes, use \"p4 revert ...\", and delete"
1461 print "the submit template file \"%s\"" % fileName
1462 if filesToAdd:
1463 print "Since the commit adds new files, they must be deleted:"
1464 for f in filesToAdd:
1465 print " " + f
1466 print
1467 return True
1470 # Let the user edit the change description, then submit it.
1472 if self.edit_template(fileName):
1473 # read the edited message and submit
1474 ret = True
1475 tmpFile = open(fileName, "rb")
1476 message = tmpFile.read()
1477 tmpFile.close()
1478 if self.isWindows:
1479 message = message.replace("\r\n", "\n")
1480 submitTemplate = message[:message.index(separatorLine)]
1481 p4_write_pipe(['submit', '-i'], submitTemplate)
1483 if self.preserveUser:
1484 if p4User:
1485 # Get last changelist number. Cannot easily get it from
1486 # the submit command output as the output is
1487 # unmarshalled.
1488 changelist = self.lastP4Changelist()
1489 self.modifyChangelistUser(changelist, p4User)
1491 # The rename/copy happened by applying a patch that created a
1492 # new file. This leaves it writable, which confuses p4.
1493 for f in pureRenameCopy:
1494 p4_sync(f, "-f")
1496 else:
1497 # skip this patch
1498 ret = False
1499 print "Submission cancelled, undoing p4 changes."
1500 for f in editedFiles:
1501 p4_revert(f)
1502 for f in filesToAdd:
1503 p4_revert(f)
1504 os.remove(f)
1505 for f in filesToDelete:
1506 p4_revert(f)
1508 os.remove(fileName)
1509 return ret
1511 # Export git tags as p4 labels. Create a p4 label and then tag
1512 # with that.
1513 def exportGitTags(self, gitTags):
1514 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1515 if len(validLabelRegexp) == 0:
1516 validLabelRegexp = defaultLabelRegexp
1517 m = re.compile(validLabelRegexp)
1519 for name in gitTags:
1521 if not m.match(name):
1522 if verbose:
1523 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1524 continue
1526 # Get the p4 commit this corresponds to
1527 logMessage = extractLogMessageFromGitCommit(name)
1528 values = extractSettingsGitLog(logMessage)
1530 if not values.has_key('change'):
1531 # a tag pointing to something not sent to p4; ignore
1532 if verbose:
1533 print "git tag %s does not give a p4 commit" % name
1534 continue
1535 else:
1536 changelist = values['change']
1538 # Get the tag details.
1539 inHeader = True
1540 isAnnotated = False
1541 body = []
1542 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1543 l = l.strip()
1544 if inHeader:
1545 if re.match(r'tag\s+', l):
1546 isAnnotated = True
1547 elif re.match(r'\s*$', l):
1548 inHeader = False
1549 continue
1550 else:
1551 body.append(l)
1553 if not isAnnotated:
1554 body = ["lightweight tag imported by git p4\n"]
1556 # Create the label - use the same view as the client spec we are using
1557 clientSpec = getClientSpec()
1559 labelTemplate = "Label: %s\n" % name
1560 labelTemplate += "Description:\n"
1561 for b in body:
1562 labelTemplate += "\t" + b + "\n"
1563 labelTemplate += "View:\n"
1564 for depot_side in clientSpec.mappings:
1565 labelTemplate += "\t%s\n" % depot_side
1567 if self.dry_run:
1568 print "Would create p4 label %s for tag" % name
1569 elif self.prepare_p4_only:
1570 print "Not creating p4 label %s for tag due to option" \
1571 " --prepare-p4-only" % name
1572 else:
1573 p4_write_pipe(["label", "-i"], labelTemplate)
1575 # Use the label
1576 p4_system(["tag", "-l", name] +
1577 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1579 if verbose:
1580 print "created p4 label for tag %s" % name
1582 def run(self, args):
1583 if len(args) == 0:
1584 self.master = currentGitBranch()
1585 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1586 die("Detecting current git branch failed!")
1587 elif len(args) == 1:
1588 self.master = args[0]
1589 if not branchExists(self.master):
1590 die("Branch %s does not exist" % self.master)
1591 else:
1592 return False
1594 allowSubmit = gitConfig("git-p4.allowSubmit")
1595 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1596 die("%s is not in git-p4.allowSubmit" % self.master)
1598 [upstream, settings] = findUpstreamBranchPoint()
1599 self.depotPath = settings['depot-paths'][0]
1600 if len(self.origin) == 0:
1601 self.origin = upstream
1603 if self.preserveUser:
1604 if not self.canChangeChangelists():
1605 die("Cannot preserve user names without p4 super-user or admin permissions")
1607 # if not set from the command line, try the config file
1608 if self.conflict_behavior is None:
1609 val = gitConfig("git-p4.conflict")
1610 if val:
1611 if val not in self.conflict_behavior_choices:
1612 die("Invalid value '%s' for config git-p4.conflict" % val)
1613 else:
1614 val = "ask"
1615 self.conflict_behavior = val
1617 if self.verbose:
1618 print "Origin branch is " + self.origin
1620 if len(self.depotPath) == 0:
1621 print "Internal error: cannot locate perforce depot path from existing branches"
1622 sys.exit(128)
1624 self.useClientSpec = False
1625 if gitConfigBool("git-p4.useclientspec"):
1626 self.useClientSpec = True
1627 if self.useClientSpec:
1628 self.clientSpecDirs = getClientSpec()
1630 if self.useClientSpec:
1631 # all files are relative to the client spec
1632 self.clientPath = getClientRoot()
1633 else:
1634 self.clientPath = p4Where(self.depotPath)
1636 if self.clientPath == "":
1637 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1639 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1640 self.oldWorkingDirectory = os.getcwd()
1642 # ensure the clientPath exists
1643 new_client_dir = False
1644 if not os.path.exists(self.clientPath):
1645 new_client_dir = True
1646 os.makedirs(self.clientPath)
1648 chdir(self.clientPath, is_client_path=True)
1649 if self.dry_run:
1650 print "Would synchronize p4 checkout in %s" % self.clientPath
1651 else:
1652 print "Synchronizing p4 checkout..."
1653 if new_client_dir:
1654 # old one was destroyed, and maybe nobody told p4
1655 p4_sync("...", "-f")
1656 else:
1657 p4_sync("...")
1658 self.check()
1660 commits = []
1661 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1662 commits.append(line.strip())
1663 commits.reverse()
1665 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1666 self.checkAuthorship = False
1667 else:
1668 self.checkAuthorship = True
1670 if self.preserveUser:
1671 self.checkValidP4Users(commits)
1674 # Build up a set of options to be passed to diff when
1675 # submitting each commit to p4.
1677 if self.detectRenames:
1678 # command-line -M arg
1679 self.diffOpts = "-M"
1680 else:
1681 # If not explicitly set check the config variable
1682 detectRenames = gitConfig("git-p4.detectRenames")
1684 if detectRenames.lower() == "false" or detectRenames == "":
1685 self.diffOpts = ""
1686 elif detectRenames.lower() == "true":
1687 self.diffOpts = "-M"
1688 else:
1689 self.diffOpts = "-M%s" % detectRenames
1691 # no command-line arg for -C or --find-copies-harder, just
1692 # config variables
1693 detectCopies = gitConfig("git-p4.detectCopies")
1694 if detectCopies.lower() == "false" or detectCopies == "":
1695 pass
1696 elif detectCopies.lower() == "true":
1697 self.diffOpts += " -C"
1698 else:
1699 self.diffOpts += " -C%s" % detectCopies
1701 if gitConfigBool("git-p4.detectCopiesHarder"):
1702 self.diffOpts += " --find-copies-harder"
1705 # Apply the commits, one at a time. On failure, ask if should
1706 # continue to try the rest of the patches, or quit.
1708 if self.dry_run:
1709 print "Would apply"
1710 applied = []
1711 last = len(commits) - 1
1712 for i, commit in enumerate(commits):
1713 if self.dry_run:
1714 print " ", read_pipe(["git", "show", "-s",
1715 "--format=format:%h %s", commit])
1716 ok = True
1717 else:
1718 ok = self.applyCommit(commit)
1719 if ok:
1720 applied.append(commit)
1721 else:
1722 if self.prepare_p4_only and i < last:
1723 print "Processing only the first commit due to option" \
1724 " --prepare-p4-only"
1725 break
1726 if i < last:
1727 quit = False
1728 while True:
1729 # prompt for what to do, or use the option/variable
1730 if self.conflict_behavior == "ask":
1731 print "What do you want to do?"
1732 response = raw_input("[s]kip this commit but apply"
1733 " the rest, or [q]uit? ")
1734 if not response:
1735 continue
1736 elif self.conflict_behavior == "skip":
1737 response = "s"
1738 elif self.conflict_behavior == "quit":
1739 response = "q"
1740 else:
1741 die("Unknown conflict_behavior '%s'" %
1742 self.conflict_behavior)
1744 if response[0] == "s":
1745 print "Skipping this commit, but applying the rest"
1746 break
1747 if response[0] == "q":
1748 print "Quitting"
1749 quit = True
1750 break
1751 if quit:
1752 break
1754 chdir(self.oldWorkingDirectory)
1756 if self.dry_run:
1757 pass
1758 elif self.prepare_p4_only:
1759 pass
1760 elif len(commits) == len(applied):
1761 print "All commits applied!"
1763 sync = P4Sync()
1764 if self.branch:
1765 sync.branch = self.branch
1766 sync.run([])
1768 rebase = P4Rebase()
1769 rebase.rebase()
1771 else:
1772 if len(applied) == 0:
1773 print "No commits applied."
1774 else:
1775 print "Applied only the commits marked with '*':"
1776 for c in commits:
1777 if c in applied:
1778 star = "*"
1779 else:
1780 star = " "
1781 print star, read_pipe(["git", "show", "-s",
1782 "--format=format:%h %s", c])
1783 print "You will have to do 'git p4 sync' and rebase."
1785 if gitConfigBool("git-p4.exportLabels"):
1786 self.exportLabels = True
1788 if self.exportLabels:
1789 p4Labels = getP4Labels(self.depotPath)
1790 gitTags = getGitTags()
1792 missingGitTags = gitTags - p4Labels
1793 self.exportGitTags(missingGitTags)
1795 # exit with error unless everything applied perfectly
1796 if len(commits) != len(applied):
1797 sys.exit(1)
1799 return True
1801 class View(object):
1802 """Represent a p4 view ("p4 help views"), and map files in a
1803 repo according to the view."""
1805 def __init__(self, client_name):
1806 self.mappings = []
1807 self.client_prefix = "//%s/" % client_name
1808 # cache results of "p4 where" to lookup client file locations
1809 self.client_spec_path_cache = {}
1811 def append(self, view_line):
1812 """Parse a view line, splitting it into depot and client
1813 sides. Append to self.mappings, preserving order. This
1814 is only needed for tag creation."""
1816 # Split the view line into exactly two words. P4 enforces
1817 # structure on these lines that simplifies this quite a bit.
1819 # Either or both words may be double-quoted.
1820 # Single quotes do not matter.
1821 # Double-quote marks cannot occur inside the words.
1822 # A + or - prefix is also inside the quotes.
1823 # There are no quotes unless they contain a space.
1824 # The line is already white-space stripped.
1825 # The two words are separated by a single space.
1827 if view_line[0] == '"':
1828 # First word is double quoted. Find its end.
1829 close_quote_index = view_line.find('"', 1)
1830 if close_quote_index <= 0:
1831 die("No first-word closing quote found: %s" % view_line)
1832 depot_side = view_line[1:close_quote_index]
1833 # skip closing quote and space
1834 rhs_index = close_quote_index + 1 + 1
1835 else:
1836 space_index = view_line.find(" ")
1837 if space_index <= 0:
1838 die("No word-splitting space found: %s" % view_line)
1839 depot_side = view_line[0:space_index]
1840 rhs_index = space_index + 1
1842 # prefix + means overlay on previous mapping
1843 if depot_side.startswith("+"):
1844 depot_side = depot_side[1:]
1846 # prefix - means exclude this path, leave out of mappings
1847 exclude = False
1848 if depot_side.startswith("-"):
1849 exclude = True
1850 depot_side = depot_side[1:]
1852 if not exclude:
1853 self.mappings.append(depot_side)
1855 def convert_client_path(self, clientFile):
1856 # chop off //client/ part to make it relative
1857 if not clientFile.startswith(self.client_prefix):
1858 die("No prefix '%s' on clientFile '%s'" %
1859 (self.client_prefix, clientFile))
1860 return clientFile[len(self.client_prefix):]
1862 def update_client_spec_path_cache(self, files):
1863 """ Caching file paths by "p4 where" batch query """
1865 # List depot file paths exclude that already cached
1866 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1868 if len(fileArgs) == 0:
1869 return # All files in cache
1871 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1872 for res in where_result:
1873 if "code" in res and res["code"] == "error":
1874 # assume error is "... file(s) not in client view"
1875 continue
1876 if "clientFile" not in res:
1877 die("No clientFile in 'p4 where' output")
1878 if "unmap" in res:
1879 # it will list all of them, but only one not unmap-ped
1880 continue
1881 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1883 # not found files or unmap files set to ""
1884 for depotFile in fileArgs:
1885 if depotFile not in self.client_spec_path_cache:
1886 self.client_spec_path_cache[depotFile] = ""
1888 def map_in_client(self, depot_path):
1889 """Return the relative location in the client where this
1890 depot file should live. Returns "" if the file should
1891 not be mapped in the client."""
1893 if depot_path in self.client_spec_path_cache:
1894 return self.client_spec_path_cache[depot_path]
1896 die( "Error: %s is not found in client spec path" % depot_path )
1897 return ""
1899 class P4Sync(Command, P4UserMap):
1900 delete_actions = ( "delete", "move/delete", "purge" )
1902 def __init__(self):
1903 Command.__init__(self)
1904 P4UserMap.__init__(self)
1905 self.options = [
1906 optparse.make_option("--branch", dest="branch"),
1907 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1908 optparse.make_option("--changesfile", dest="changesFile"),
1909 optparse.make_option("--silent", dest="silent", action="store_true"),
1910 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1911 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1912 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1913 help="Import into refs/heads/ , not refs/remotes"),
1914 optparse.make_option("--max-changes", dest="maxChanges"),
1915 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1916 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1917 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1918 help="Only sync files that are included in the Perforce Client Spec"),
1919 optparse.make_option("-/", dest="cloneExclude",
1920 action="append", type="string",
1921 help="exclude depot path"),
1923 self.description = """Imports from Perforce into a git repository.\n
1924 example:
1925 //depot/my/project/ -- to import the current head
1926 //depot/my/project/@all -- to import everything
1927 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1929 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1931 self.usage += " //depot/path[@revRange]"
1932 self.silent = False
1933 self.createdBranches = set()
1934 self.committedChanges = set()
1935 self.branch = ""
1936 self.detectBranches = False
1937 self.detectLabels = False
1938 self.importLabels = False
1939 self.changesFile = ""
1940 self.syncWithOrigin = True
1941 self.importIntoRemotes = True
1942 self.maxChanges = ""
1943 self.keepRepoPath = False
1944 self.depotPaths = None
1945 self.p4BranchesInGit = []
1946 self.cloneExclude = []
1947 self.useClientSpec = False
1948 self.useClientSpec_from_options = False
1949 self.clientSpecDirs = None
1950 self.tempBranches = []
1951 self.tempBranchLocation = "git-p4-tmp"
1953 if gitConfig("git-p4.syncFromOrigin") == "false":
1954 self.syncWithOrigin = False
1956 # This is required for the "append" cloneExclude action
1957 def ensure_value(self, attr, value):
1958 if not hasattr(self, attr) or getattr(self, attr) is None:
1959 setattr(self, attr, value)
1960 return getattr(self, attr)
1962 # Force a checkpoint in fast-import and wait for it to finish
1963 def checkpoint(self):
1964 self.gitStream.write("checkpoint\n\n")
1965 self.gitStream.write("progress checkpoint\n\n")
1966 out = self.gitOutput.readline()
1967 if self.verbose:
1968 print "checkpoint finished: " + out
1970 def extractFilesFromCommit(self, commit):
1971 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1972 for path in self.cloneExclude]
1973 files = []
1974 fnum = 0
1975 while commit.has_key("depotFile%s" % fnum):
1976 path = commit["depotFile%s" % fnum]
1978 if [p for p in self.cloneExclude
1979 if p4PathStartsWith(path, p)]:
1980 found = False
1981 else:
1982 found = [p for p in self.depotPaths
1983 if p4PathStartsWith(path, p)]
1984 if not found:
1985 fnum = fnum + 1
1986 continue
1988 file = {}
1989 file["path"] = path
1990 file["rev"] = commit["rev%s" % fnum]
1991 file["action"] = commit["action%s" % fnum]
1992 file["type"] = commit["type%s" % fnum]
1993 files.append(file)
1994 fnum = fnum + 1
1995 return files
1997 def stripRepoPath(self, path, prefixes):
1998 """When streaming files, this is called to map a p4 depot path
1999 to where it should go in git. The prefixes are either
2000 self.depotPaths, or self.branchPrefixes in the case of
2001 branch detection."""
2003 if self.useClientSpec:
2004 # branch detection moves files up a level (the branch name)
2005 # from what client spec interpretation gives
2006 path = self.clientSpecDirs.map_in_client(path)
2007 if self.detectBranches:
2008 for b in self.knownBranches:
2009 if path.startswith(b + "/"):
2010 path = path[len(b)+1:]
2012 elif self.keepRepoPath:
2013 # Preserve everything in relative path name except leading
2014 # //depot/; just look at first prefix as they all should
2015 # be in the same depot.
2016 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2017 if p4PathStartsWith(path, depot):
2018 path = path[len(depot):]
2020 else:
2021 for p in prefixes:
2022 if p4PathStartsWith(path, p):
2023 path = path[len(p):]
2024 break
2026 path = wildcard_decode(path)
2027 return path
2029 def splitFilesIntoBranches(self, commit):
2030 """Look at each depotFile in the commit to figure out to what
2031 branch it belongs."""
2033 if self.clientSpecDirs:
2034 files = self.extractFilesFromCommit(commit)
2035 self.clientSpecDirs.update_client_spec_path_cache(files)
2037 branches = {}
2038 fnum = 0
2039 while commit.has_key("depotFile%s" % fnum):
2040 path = commit["depotFile%s" % fnum]
2041 found = [p for p in self.depotPaths
2042 if p4PathStartsWith(path, p)]
2043 if not found:
2044 fnum = fnum + 1
2045 continue
2047 file = {}
2048 file["path"] = path
2049 file["rev"] = commit["rev%s" % fnum]
2050 file["action"] = commit["action%s" % fnum]
2051 file["type"] = commit["type%s" % fnum]
2052 fnum = fnum + 1
2054 # start with the full relative path where this file would
2055 # go in a p4 client
2056 if self.useClientSpec:
2057 relPath = self.clientSpecDirs.map_in_client(path)
2058 else:
2059 relPath = self.stripRepoPath(path, self.depotPaths)
2061 for branch in self.knownBranches.keys():
2062 # add a trailing slash so that a commit into qt/4.2foo
2063 # doesn't end up in qt/4.2, e.g.
2064 if relPath.startswith(branch + "/"):
2065 if branch not in branches:
2066 branches[branch] = []
2067 branches[branch].append(file)
2068 break
2070 return branches
2072 # output one file from the P4 stream
2073 # - helper for streamP4Files
2075 def streamOneP4File(self, file, contents):
2076 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2077 if verbose:
2078 sys.stderr.write("%s\n" % relPath)
2080 (type_base, type_mods) = split_p4_type(file["type"])
2082 git_mode = "100644"
2083 if "x" in type_mods:
2084 git_mode = "100755"
2085 if type_base == "symlink":
2086 git_mode = "120000"
2087 # p4 print on a symlink sometimes contains "target\n";
2088 # if it does, remove the newline
2089 data = ''.join(contents)
2090 if not data:
2091 # Some version of p4 allowed creating a symlink that pointed
2092 # to nothing. This causes p4 errors when checking out such
2093 # a change, and errors here too. Work around it by ignoring
2094 # the bad symlink; hopefully a future change fixes it.
2095 print "\nIgnoring empty symlink in %s" % file['depotFile']
2096 return
2097 elif data[-1] == '\n':
2098 contents = [data[:-1]]
2099 else:
2100 contents = [data]
2102 if type_base == "utf16":
2103 # p4 delivers different text in the python output to -G
2104 # than it does when using "print -o", or normal p4 client
2105 # operations. utf16 is converted to ascii or utf8, perhaps.
2106 # But ascii text saved as -t utf16 is completely mangled.
2107 # Invoke print -o to get the real contents.
2109 # On windows, the newlines will always be mangled by print, so put
2110 # them back too. This is not needed to the cygwin windows version,
2111 # just the native "NT" type.
2113 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2114 if p4_version_string().find("/NT") >= 0:
2115 text = text.replace("\r\n", "\n")
2116 contents = [ text ]
2118 if type_base == "apple":
2119 # Apple filetype files will be streamed as a concatenation of
2120 # its appledouble header and the contents. This is useless
2121 # on both macs and non-macs. If using "print -q -o xx", it
2122 # will create "xx" with the data, and "%xx" with the header.
2123 # This is also not very useful.
2125 # Ideally, someday, this script can learn how to generate
2126 # appledouble files directly and import those to git, but
2127 # non-mac machines can never find a use for apple filetype.
2128 print "\nIgnoring apple filetype file %s" % file['depotFile']
2129 return
2131 # Note that we do not try to de-mangle keywords on utf16 files,
2132 # even though in theory somebody may want that.
2133 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2134 if pattern:
2135 regexp = re.compile(pattern, re.VERBOSE)
2136 text = ''.join(contents)
2137 text = regexp.sub(r'$\1$', text)
2138 contents = [ text ]
2140 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2142 # total length...
2143 length = 0
2144 for d in contents:
2145 length = length + len(d)
2147 self.gitStream.write("data %d\n" % length)
2148 for d in contents:
2149 self.gitStream.write(d)
2150 self.gitStream.write("\n")
2152 def streamOneP4Deletion(self, file):
2153 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2154 if verbose:
2155 sys.stderr.write("delete %s\n" % relPath)
2156 self.gitStream.write("D %s\n" % relPath)
2158 # handle another chunk of streaming data
2159 def streamP4FilesCb(self, marshalled):
2161 # catch p4 errors and complain
2162 err = None
2163 if "code" in marshalled:
2164 if marshalled["code"] == "error":
2165 if "data" in marshalled:
2166 err = marshalled["data"].rstrip()
2167 if err:
2168 f = None
2169 if self.stream_have_file_info:
2170 if "depotFile" in self.stream_file:
2171 f = self.stream_file["depotFile"]
2172 # force a failure in fast-import, else an empty
2173 # commit will be made
2174 self.gitStream.write("\n")
2175 self.gitStream.write("die-now\n")
2176 self.gitStream.close()
2177 # ignore errors, but make sure it exits first
2178 self.importProcess.wait()
2179 if f:
2180 die("Error from p4 print for %s: %s" % (f, err))
2181 else:
2182 die("Error from p4 print: %s" % err)
2184 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2185 # start of a new file - output the old one first
2186 self.streamOneP4File(self.stream_file, self.stream_contents)
2187 self.stream_file = {}
2188 self.stream_contents = []
2189 self.stream_have_file_info = False
2191 # pick up the new file information... for the
2192 # 'data' field we need to append to our array
2193 for k in marshalled.keys():
2194 if k == 'data':
2195 self.stream_contents.append(marshalled['data'])
2196 else:
2197 self.stream_file[k] = marshalled[k]
2199 self.stream_have_file_info = True
2201 # Stream directly from "p4 files" into "git fast-import"
2202 def streamP4Files(self, files):
2203 filesForCommit = []
2204 filesToRead = []
2205 filesToDelete = []
2207 for f in files:
2208 # if using a client spec, only add the files that have
2209 # a path in the client
2210 if self.clientSpecDirs:
2211 if self.clientSpecDirs.map_in_client(f['path']) == "":
2212 continue
2214 filesForCommit.append(f)
2215 if f['action'] in self.delete_actions:
2216 filesToDelete.append(f)
2217 else:
2218 filesToRead.append(f)
2220 # deleted files...
2221 for f in filesToDelete:
2222 self.streamOneP4Deletion(f)
2224 if len(filesToRead) > 0:
2225 self.stream_file = {}
2226 self.stream_contents = []
2227 self.stream_have_file_info = False
2229 # curry self argument
2230 def streamP4FilesCbSelf(entry):
2231 self.streamP4FilesCb(entry)
2233 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2235 p4CmdList(["-x", "-", "print"],
2236 stdin=fileArgs,
2237 cb=streamP4FilesCbSelf)
2239 # do the last chunk
2240 if self.stream_file.has_key('depotFile'):
2241 self.streamOneP4File(self.stream_file, self.stream_contents)
2243 def make_email(self, userid):
2244 if userid in self.users:
2245 return self.users[userid]
2246 else:
2247 return "%s <a@b>" % userid
2249 # Stream a p4 tag
2250 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2251 if verbose:
2252 print "writing tag %s for commit %s" % (labelName, commit)
2253 gitStream.write("tag %s\n" % labelName)
2254 gitStream.write("from %s\n" % commit)
2256 if labelDetails.has_key('Owner'):
2257 owner = labelDetails["Owner"]
2258 else:
2259 owner = None
2261 # Try to use the owner of the p4 label, or failing that,
2262 # the current p4 user id.
2263 if owner:
2264 email = self.make_email(owner)
2265 else:
2266 email = self.make_email(self.p4UserId())
2267 tagger = "%s %s %s" % (email, epoch, self.tz)
2269 gitStream.write("tagger %s\n" % tagger)
2271 print "labelDetails=",labelDetails
2272 if labelDetails.has_key('Description'):
2273 description = labelDetails['Description']
2274 else:
2275 description = 'Label from git p4'
2277 gitStream.write("data %d\n" % len(description))
2278 gitStream.write(description)
2279 gitStream.write("\n")
2281 def commit(self, details, files, branch, parent = ""):
2282 epoch = details["time"]
2283 author = details["user"]
2285 if self.verbose:
2286 print "commit into %s" % branch
2288 # start with reading files; if that fails, we should not
2289 # create a commit.
2290 new_files = []
2291 for f in files:
2292 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2293 new_files.append (f)
2294 else:
2295 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2297 if self.clientSpecDirs:
2298 self.clientSpecDirs.update_client_spec_path_cache(files)
2300 self.gitStream.write("commit %s\n" % branch)
2301 # gitStream.write("mark :%s\n" % details["change"])
2302 self.committedChanges.add(int(details["change"]))
2303 committer = ""
2304 if author not in self.users:
2305 self.getUserMapFromPerforceServer()
2306 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2308 self.gitStream.write("committer %s\n" % committer)
2310 self.gitStream.write("data <<EOT\n")
2311 self.gitStream.write(details["desc"])
2312 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2313 (','.join(self.branchPrefixes), details["change"]))
2314 if len(details['options']) > 0:
2315 self.gitStream.write(": options = %s" % details['options'])
2316 self.gitStream.write("]\nEOT\n\n")
2318 if len(parent) > 0:
2319 if self.verbose:
2320 print "parent %s" % parent
2321 self.gitStream.write("from %s\n" % parent)
2323 self.streamP4Files(new_files)
2324 self.gitStream.write("\n")
2326 change = int(details["change"])
2328 if self.labels.has_key(change):
2329 label = self.labels[change]
2330 labelDetails = label[0]
2331 labelRevisions = label[1]
2332 if self.verbose:
2333 print "Change %s is labelled %s" % (change, labelDetails)
2335 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2336 for p in self.branchPrefixes])
2338 if len(files) == len(labelRevisions):
2340 cleanedFiles = {}
2341 for info in files:
2342 if info["action"] in self.delete_actions:
2343 continue
2344 cleanedFiles[info["depotFile"]] = info["rev"]
2346 if cleanedFiles == labelRevisions:
2347 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2349 else:
2350 if not self.silent:
2351 print ("Tag %s does not match with change %s: files do not match."
2352 % (labelDetails["label"], change))
2354 else:
2355 if not self.silent:
2356 print ("Tag %s does not match with change %s: file count is different."
2357 % (labelDetails["label"], change))
2359 # Build a dictionary of changelists and labels, for "detect-labels" option.
2360 def getLabels(self):
2361 self.labels = {}
2363 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2364 if len(l) > 0 and not self.silent:
2365 print "Finding files belonging to labels in %s" % `self.depotPaths`
2367 for output in l:
2368 label = output["label"]
2369 revisions = {}
2370 newestChange = 0
2371 if self.verbose:
2372 print "Querying files for label %s" % label
2373 for file in p4CmdList(["files"] +
2374 ["%s...@%s" % (p, label)
2375 for p in self.depotPaths]):
2376 revisions[file["depotFile"]] = file["rev"]
2377 change = int(file["change"])
2378 if change > newestChange:
2379 newestChange = change
2381 self.labels[newestChange] = [output, revisions]
2383 if self.verbose:
2384 print "Label changes: %s" % self.labels.keys()
2386 # Import p4 labels as git tags. A direct mapping does not
2387 # exist, so assume that if all the files are at the same revision
2388 # then we can use that, or it's something more complicated we should
2389 # just ignore.
2390 def importP4Labels(self, stream, p4Labels):
2391 if verbose:
2392 print "import p4 labels: " + ' '.join(p4Labels)
2394 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2395 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2396 if len(validLabelRegexp) == 0:
2397 validLabelRegexp = defaultLabelRegexp
2398 m = re.compile(validLabelRegexp)
2400 for name in p4Labels:
2401 commitFound = False
2403 if not m.match(name):
2404 if verbose:
2405 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2406 continue
2408 if name in ignoredP4Labels:
2409 continue
2411 labelDetails = p4CmdList(['label', "-o", name])[0]
2413 # get the most recent changelist for each file in this label
2414 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2415 for p in self.depotPaths])
2417 if change.has_key('change'):
2418 # find the corresponding git commit; take the oldest commit
2419 changelist = int(change['change'])
2420 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2421 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2422 if len(gitCommit) == 0:
2423 print "could not find git commit for changelist %d" % changelist
2424 else:
2425 gitCommit = gitCommit.strip()
2426 commitFound = True
2427 # Convert from p4 time format
2428 try:
2429 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2430 except ValueError:
2431 print "Could not convert label time %s" % labelDetails['Update']
2432 tmwhen = 1
2434 when = int(time.mktime(tmwhen))
2435 self.streamTag(stream, name, labelDetails, gitCommit, when)
2436 if verbose:
2437 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2438 else:
2439 if verbose:
2440 print "Label %s has no changelists - possibly deleted?" % name
2442 if not commitFound:
2443 # We can't import this label; don't try again as it will get very
2444 # expensive repeatedly fetching all the files for labels that will
2445 # never be imported. If the label is moved in the future, the
2446 # ignore will need to be removed manually.
2447 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2449 def guessProjectName(self):
2450 for p in self.depotPaths:
2451 if p.endswith("/"):
2452 p = p[:-1]
2453 p = p[p.strip().rfind("/") + 1:]
2454 if not p.endswith("/"):
2455 p += "/"
2456 return p
2458 def getBranchMapping(self):
2459 lostAndFoundBranches = set()
2461 user = gitConfig("git-p4.branchUser")
2462 if len(user) > 0:
2463 command = "branches -u %s" % user
2464 else:
2465 command = "branches"
2467 for info in p4CmdList(command):
2468 details = p4Cmd(["branch", "-o", info["branch"]])
2469 viewIdx = 0
2470 while details.has_key("View%s" % viewIdx):
2471 paths = details["View%s" % viewIdx].split(" ")
2472 viewIdx = viewIdx + 1
2473 # require standard //depot/foo/... //depot/bar/... mapping
2474 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2475 continue
2476 source = paths[0]
2477 destination = paths[1]
2478 ## HACK
2479 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2480 source = source[len(self.depotPaths[0]):-4]
2481 destination = destination[len(self.depotPaths[0]):-4]
2483 if destination in self.knownBranches:
2484 if not self.silent:
2485 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2486 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2487 continue
2489 self.knownBranches[destination] = source
2491 lostAndFoundBranches.discard(destination)
2493 if source not in self.knownBranches:
2494 lostAndFoundBranches.add(source)
2496 # Perforce does not strictly require branches to be defined, so we also
2497 # check git config for a branch list.
2499 # Example of branch definition in git config file:
2500 # [git-p4]
2501 # branchList=main:branchA
2502 # branchList=main:branchB
2503 # branchList=branchA:branchC
2504 configBranches = gitConfigList("git-p4.branchList")
2505 for branch in configBranches:
2506 if branch:
2507 (source, destination) = branch.split(":")
2508 self.knownBranches[destination] = source
2510 lostAndFoundBranches.discard(destination)
2512 if source not in self.knownBranches:
2513 lostAndFoundBranches.add(source)
2516 for branch in lostAndFoundBranches:
2517 self.knownBranches[branch] = branch
2519 def getBranchMappingFromGitBranches(self):
2520 branches = p4BranchesInGit(self.importIntoRemotes)
2521 for branch in branches.keys():
2522 if branch == "master":
2523 branch = "main"
2524 else:
2525 branch = branch[len(self.projectName):]
2526 self.knownBranches[branch] = branch
2528 def updateOptionDict(self, d):
2529 option_keys = {}
2530 if self.keepRepoPath:
2531 option_keys['keepRepoPath'] = 1
2533 d["options"] = ' '.join(sorted(option_keys.keys()))
2535 def readOptions(self, d):
2536 self.keepRepoPath = (d.has_key('options')
2537 and ('keepRepoPath' in d['options']))
2539 def gitRefForBranch(self, branch):
2540 if branch == "main":
2541 return self.refPrefix + "master"
2543 if len(branch) <= 0:
2544 return branch
2546 return self.refPrefix + self.projectName + branch
2548 def gitCommitByP4Change(self, ref, change):
2549 if self.verbose:
2550 print "looking in ref " + ref + " for change %s using bisect..." % change
2552 earliestCommit = ""
2553 latestCommit = parseRevision(ref)
2555 while True:
2556 if self.verbose:
2557 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2558 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2559 if len(next) == 0:
2560 if self.verbose:
2561 print "argh"
2562 return ""
2563 log = extractLogMessageFromGitCommit(next)
2564 settings = extractSettingsGitLog(log)
2565 currentChange = int(settings['change'])
2566 if self.verbose:
2567 print "current change %s" % currentChange
2569 if currentChange == change:
2570 if self.verbose:
2571 print "found %s" % next
2572 return next
2574 if currentChange < change:
2575 earliestCommit = "^%s" % next
2576 else:
2577 latestCommit = "%s" % next
2579 return ""
2581 def importNewBranch(self, branch, maxChange):
2582 # make fast-import flush all changes to disk and update the refs using the checkpoint
2583 # command so that we can try to find the branch parent in the git history
2584 self.gitStream.write("checkpoint\n\n");
2585 self.gitStream.flush();
2586 branchPrefix = self.depotPaths[0] + branch + "/"
2587 range = "@1,%s" % maxChange
2588 #print "prefix" + branchPrefix
2589 changes = p4ChangesForPaths([branchPrefix], range)
2590 if len(changes) <= 0:
2591 return False
2592 firstChange = changes[0]
2593 #print "first change in branch: %s" % firstChange
2594 sourceBranch = self.knownBranches[branch]
2595 sourceDepotPath = self.depotPaths[0] + sourceBranch
2596 sourceRef = self.gitRefForBranch(sourceBranch)
2597 #print "source " + sourceBranch
2599 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2600 #print "branch parent: %s" % branchParentChange
2601 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2602 if len(gitParent) > 0:
2603 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2604 #print "parent git commit: %s" % gitParent
2606 self.importChanges(changes)
2607 return True
2609 def searchParent(self, parent, branch, target):
2610 parentFound = False
2611 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2612 "--no-merges", parent]):
2613 blob = blob.strip()
2614 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2615 parentFound = True
2616 if self.verbose:
2617 print "Found parent of %s in commit %s" % (branch, blob)
2618 break
2619 if parentFound:
2620 return blob
2621 else:
2622 return None
2624 def importChanges(self, changes):
2625 cnt = 1
2626 for change in changes:
2627 description = p4_describe(change)
2628 self.updateOptionDict(description)
2630 if not self.silent:
2631 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2632 sys.stdout.flush()
2633 cnt = cnt + 1
2635 try:
2636 if self.detectBranches:
2637 branches = self.splitFilesIntoBranches(description)
2638 for branch in branches.keys():
2639 ## HACK --hwn
2640 branchPrefix = self.depotPaths[0] + branch + "/"
2641 self.branchPrefixes = [ branchPrefix ]
2643 parent = ""
2645 filesForCommit = branches[branch]
2647 if self.verbose:
2648 print "branch is %s" % branch
2650 self.updatedBranches.add(branch)
2652 if branch not in self.createdBranches:
2653 self.createdBranches.add(branch)
2654 parent = self.knownBranches[branch]
2655 if parent == branch:
2656 parent = ""
2657 else:
2658 fullBranch = self.projectName + branch
2659 if fullBranch not in self.p4BranchesInGit:
2660 if not self.silent:
2661 print("\n Importing new branch %s" % fullBranch);
2662 if self.importNewBranch(branch, change - 1):
2663 parent = ""
2664 self.p4BranchesInGit.append(fullBranch)
2665 if not self.silent:
2666 print("\n Resuming with change %s" % change);
2668 if self.verbose:
2669 print "parent determined through known branches: %s" % parent
2671 branch = self.gitRefForBranch(branch)
2672 parent = self.gitRefForBranch(parent)
2674 if self.verbose:
2675 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2677 if len(parent) == 0 and branch in self.initialParents:
2678 parent = self.initialParents[branch]
2679 del self.initialParents[branch]
2681 blob = None
2682 if len(parent) > 0:
2683 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2684 if self.verbose:
2685 print "Creating temporary branch: " + tempBranch
2686 self.commit(description, filesForCommit, tempBranch)
2687 self.tempBranches.append(tempBranch)
2688 self.checkpoint()
2689 blob = self.searchParent(parent, branch, tempBranch)
2690 if blob:
2691 self.commit(description, filesForCommit, branch, blob)
2692 else:
2693 if self.verbose:
2694 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2695 self.commit(description, filesForCommit, branch, parent)
2696 else:
2697 files = self.extractFilesFromCommit(description)
2698 self.commit(description, files, self.branch,
2699 self.initialParent)
2700 # only needed once, to connect to the previous commit
2701 self.initialParent = ""
2702 except IOError:
2703 print self.gitError.read()
2704 sys.exit(1)
2706 def importHeadRevision(self, revision):
2707 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2709 details = {}
2710 details["user"] = "git perforce import user"
2711 details["desc"] = ("Initial import of %s from the state at revision %s\n"
2712 % (' '.join(self.depotPaths), revision))
2713 details["change"] = revision
2714 newestRevision = 0
2716 fileCnt = 0
2717 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2719 for info in p4CmdList(["files"] + fileArgs):
2721 if 'code' in info and info['code'] == 'error':
2722 sys.stderr.write("p4 returned an error: %s\n"
2723 % info['data'])
2724 if info['data'].find("must refer to client") >= 0:
2725 sys.stderr.write("This particular p4 error is misleading.\n")
2726 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2727 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
2728 sys.exit(1)
2729 if 'p4ExitCode' in info:
2730 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2731 sys.exit(1)
2734 change = int(info["change"])
2735 if change > newestRevision:
2736 newestRevision = change
2738 if info["action"] in self.delete_actions:
2739 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2740 #fileCnt = fileCnt + 1
2741 continue
2743 for prop in ["depotFile", "rev", "action", "type" ]:
2744 details["%s%s" % (prop, fileCnt)] = info[prop]
2746 fileCnt = fileCnt + 1
2748 details["change"] = newestRevision
2750 # Use time from top-most change so that all git p4 clones of
2751 # the same p4 repo have the same commit SHA1s.
2752 res = p4_describe(newestRevision)
2753 details["time"] = res["time"]
2755 self.updateOptionDict(details)
2756 try:
2757 self.commit(details, self.extractFilesFromCommit(details), self.branch)
2758 except IOError:
2759 print "IO error with git fast-import. Is your git version recent enough?"
2760 print self.gitError.read()
2763 def run(self, args):
2764 self.depotPaths = []
2765 self.changeRange = ""
2766 self.previousDepotPaths = []
2767 self.hasOrigin = False
2769 # map from branch depot path to parent branch
2770 self.knownBranches = {}
2771 self.initialParents = {}
2773 if self.importIntoRemotes:
2774 self.refPrefix = "refs/remotes/p4/"
2775 else:
2776 self.refPrefix = "refs/heads/p4/"
2778 if self.syncWithOrigin:
2779 self.hasOrigin = originP4BranchesExist()
2780 if self.hasOrigin:
2781 if not self.silent:
2782 print 'Syncing with origin first, using "git fetch origin"'
2783 system("git fetch origin")
2785 branch_arg_given = bool(self.branch)
2786 if len(self.branch) == 0:
2787 self.branch = self.refPrefix + "master"
2788 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2789 system("git update-ref %s refs/heads/p4" % self.branch)
2790 system("git branch -D p4")
2792 # accept either the command-line option, or the configuration variable
2793 if self.useClientSpec:
2794 # will use this after clone to set the variable
2795 self.useClientSpec_from_options = True
2796 else:
2797 if gitConfigBool("git-p4.useclientspec"):
2798 self.useClientSpec = True
2799 if self.useClientSpec:
2800 self.clientSpecDirs = getClientSpec()
2802 # TODO: should always look at previous commits,
2803 # merge with previous imports, if possible.
2804 if args == []:
2805 if self.hasOrigin:
2806 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2808 # branches holds mapping from branch name to sha1
2809 branches = p4BranchesInGit(self.importIntoRemotes)
2811 # restrict to just this one, disabling detect-branches
2812 if branch_arg_given:
2813 short = self.branch.split("/")[-1]
2814 if short in branches:
2815 self.p4BranchesInGit = [ short ]
2816 else:
2817 self.p4BranchesInGit = branches.keys()
2819 if len(self.p4BranchesInGit) > 1:
2820 if not self.silent:
2821 print "Importing from/into multiple branches"
2822 self.detectBranches = True
2823 for branch in branches.keys():
2824 self.initialParents[self.refPrefix + branch] = \
2825 branches[branch]
2827 if self.verbose:
2828 print "branches: %s" % self.p4BranchesInGit
2830 p4Change = 0
2831 for branch in self.p4BranchesInGit:
2832 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
2834 settings = extractSettingsGitLog(logMsg)
2836 self.readOptions(settings)
2837 if (settings.has_key('depot-paths')
2838 and settings.has_key ('change')):
2839 change = int(settings['change']) + 1
2840 p4Change = max(p4Change, change)
2842 depotPaths = sorted(settings['depot-paths'])
2843 if self.previousDepotPaths == []:
2844 self.previousDepotPaths = depotPaths
2845 else:
2846 paths = []
2847 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2848 prev_list = prev.split("/")
2849 cur_list = cur.split("/")
2850 for i in range(0, min(len(cur_list), len(prev_list))):
2851 if cur_list[i] <> prev_list[i]:
2852 i = i - 1
2853 break
2855 paths.append ("/".join(cur_list[:i + 1]))
2857 self.previousDepotPaths = paths
2859 if p4Change > 0:
2860 self.depotPaths = sorted(self.previousDepotPaths)
2861 self.changeRange = "@%s,#head" % p4Change
2862 if not self.silent and not self.detectBranches:
2863 print "Performing incremental import into %s git branch" % self.branch
2865 # accept multiple ref name abbreviations:
2866 # refs/foo/bar/branch -> use it exactly
2867 # p4/branch -> prepend refs/remotes/ or refs/heads/
2868 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2869 if not self.branch.startswith("refs/"):
2870 if self.importIntoRemotes:
2871 prepend = "refs/remotes/"
2872 else:
2873 prepend = "refs/heads/"
2874 if not self.branch.startswith("p4/"):
2875 prepend += "p4/"
2876 self.branch = prepend + self.branch
2878 if len(args) == 0 and self.depotPaths:
2879 if not self.silent:
2880 print "Depot paths: %s" % ' '.join(self.depotPaths)
2881 else:
2882 if self.depotPaths and self.depotPaths != args:
2883 print ("previous import used depot path %s and now %s was specified. "
2884 "This doesn't work!" % (' '.join (self.depotPaths),
2885 ' '.join (args)))
2886 sys.exit(1)
2888 self.depotPaths = sorted(args)
2890 revision = ""
2891 self.users = {}
2893 # Make sure no revision specifiers are used when --changesfile
2894 # is specified.
2895 bad_changesfile = False
2896 if len(self.changesFile) > 0:
2897 for p in self.depotPaths:
2898 if p.find("@") >= 0 or p.find("#") >= 0:
2899 bad_changesfile = True
2900 break
2901 if bad_changesfile:
2902 die("Option --changesfile is incompatible with revision specifiers")
2904 newPaths = []
2905 for p in self.depotPaths:
2906 if p.find("@") != -1:
2907 atIdx = p.index("@")
2908 self.changeRange = p[atIdx:]
2909 if self.changeRange == "@all":
2910 self.changeRange = ""
2911 elif ',' not in self.changeRange:
2912 revision = self.changeRange
2913 self.changeRange = ""
2914 p = p[:atIdx]
2915 elif p.find("#") != -1:
2916 hashIdx = p.index("#")
2917 revision = p[hashIdx:]
2918 p = p[:hashIdx]
2919 elif self.previousDepotPaths == []:
2920 # pay attention to changesfile, if given, else import
2921 # the entire p4 tree at the head revision
2922 if len(self.changesFile) == 0:
2923 revision = "#head"
2925 p = re.sub ("\.\.\.$", "", p)
2926 if not p.endswith("/"):
2927 p += "/"
2929 newPaths.append(p)
2931 self.depotPaths = newPaths
2933 # --detect-branches may change this for each branch
2934 self.branchPrefixes = self.depotPaths
2936 self.loadUserMapFromCache()
2937 self.labels = {}
2938 if self.detectLabels:
2939 self.getLabels();
2941 if self.detectBranches:
2942 ## FIXME - what's a P4 projectName ?
2943 self.projectName = self.guessProjectName()
2945 if self.hasOrigin:
2946 self.getBranchMappingFromGitBranches()
2947 else:
2948 self.getBranchMapping()
2949 if self.verbose:
2950 print "p4-git branches: %s" % self.p4BranchesInGit
2951 print "initial parents: %s" % self.initialParents
2952 for b in self.p4BranchesInGit:
2953 if b != "master":
2955 ## FIXME
2956 b = b[len(self.projectName):]
2957 self.createdBranches.add(b)
2959 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2961 self.importProcess = subprocess.Popen(["git", "fast-import"],
2962 stdin=subprocess.PIPE,
2963 stdout=subprocess.PIPE,
2964 stderr=subprocess.PIPE);
2965 self.gitOutput = self.importProcess.stdout
2966 self.gitStream = self.importProcess.stdin
2967 self.gitError = self.importProcess.stderr
2969 if revision:
2970 self.importHeadRevision(revision)
2971 else:
2972 changes = []
2974 if len(self.changesFile) > 0:
2975 output = open(self.changesFile).readlines()
2976 changeSet = set()
2977 for line in output:
2978 changeSet.add(int(line))
2980 for change in changeSet:
2981 changes.append(change)
2983 changes.sort()
2984 else:
2985 # catch "git p4 sync" with no new branches, in a repo that
2986 # does not have any existing p4 branches
2987 if len(args) == 0:
2988 if not self.p4BranchesInGit:
2989 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
2991 # The default branch is master, unless --branch is used to
2992 # specify something else. Make sure it exists, or complain
2993 # nicely about how to use --branch.
2994 if not self.detectBranches:
2995 if not branch_exists(self.branch):
2996 if branch_arg_given:
2997 die("Error: branch %s does not exist." % self.branch)
2998 else:
2999 die("Error: no branch %s; perhaps specify one with --branch." %
3000 self.branch)
3002 if self.verbose:
3003 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3004 self.changeRange)
3005 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
3007 if len(self.maxChanges) > 0:
3008 changes = changes[:min(int(self.maxChanges), len(changes))]
3010 if len(changes) == 0:
3011 if not self.silent:
3012 print "No changes to import!"
3013 else:
3014 if not self.silent and not self.detectBranches:
3015 print "Import destination: %s" % self.branch
3017 self.updatedBranches = set()
3019 if not self.detectBranches:
3020 if args:
3021 # start a new branch
3022 self.initialParent = ""
3023 else:
3024 # build on a previous revision
3025 self.initialParent = parseRevision(self.branch)
3027 self.importChanges(changes)
3029 if not self.silent:
3030 print ""
3031 if len(self.updatedBranches) > 0:
3032 sys.stdout.write("Updated branches: ")
3033 for b in self.updatedBranches:
3034 sys.stdout.write("%s " % b)
3035 sys.stdout.write("\n")
3037 if gitConfigBool("git-p4.importLabels"):
3038 self.importLabels = True
3040 if self.importLabels:
3041 p4Labels = getP4Labels(self.depotPaths)
3042 gitTags = getGitTags()
3044 missingP4Labels = p4Labels - gitTags
3045 self.importP4Labels(self.gitStream, missingP4Labels)
3047 self.gitStream.close()
3048 if self.importProcess.wait() != 0:
3049 die("fast-import failed: %s" % self.gitError.read())
3050 self.gitOutput.close()
3051 self.gitError.close()
3053 # Cleanup temporary branches created during import
3054 if self.tempBranches != []:
3055 for branch in self.tempBranches:
3056 read_pipe("git update-ref -d %s" % branch)
3057 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3059 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3060 # a convenient shortcut refname "p4".
3061 if self.importIntoRemotes:
3062 head_ref = self.refPrefix + "HEAD"
3063 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3064 system(["git", "symbolic-ref", head_ref, self.branch])
3066 return True
3068 class P4Rebase(Command):
3069 def __init__(self):
3070 Command.__init__(self)
3071 self.options = [
3072 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3074 self.importLabels = False
3075 self.description = ("Fetches the latest revision from perforce and "
3076 + "rebases the current work (branch) against it")
3078 def run(self, args):
3079 sync = P4Sync()
3080 sync.importLabels = self.importLabels
3081 sync.run([])
3083 return self.rebase()
3085 def rebase(self):
3086 if os.system("git update-index --refresh") != 0:
3087 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3088 if len(read_pipe("git diff-index HEAD --")) > 0:
3089 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3091 [upstream, settings] = findUpstreamBranchPoint()
3092 if len(upstream) == 0:
3093 die("Cannot find upstream branchpoint for rebase")
3095 # the branchpoint may be p4/foo~3, so strip off the parent
3096 upstream = re.sub("~[0-9]+$", "", upstream)
3098 print "Rebasing the current branch onto %s" % upstream
3099 oldHead = read_pipe("git rev-parse HEAD").strip()
3100 system("git rebase %s" % upstream)
3101 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3102 return True
3104 class P4Clone(P4Sync):
3105 def __init__(self):
3106 P4Sync.__init__(self)
3107 self.description = "Creates a new git repository and imports from Perforce into it"
3108 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3109 self.options += [
3110 optparse.make_option("--destination", dest="cloneDestination",
3111 action='store', default=None,
3112 help="where to leave result of the clone"),
3113 optparse.make_option("--bare", dest="cloneBare",
3114 action="store_true", default=False),
3116 self.cloneDestination = None
3117 self.needsGit = False
3118 self.cloneBare = False
3120 def defaultDestination(self, args):
3121 ## TODO: use common prefix of args?
3122 depotPath = args[0]
3123 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3124 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3125 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3126 depotDir = re.sub(r"/$", "", depotDir)
3127 return os.path.split(depotDir)[1]
3129 def run(self, args):
3130 if len(args) < 1:
3131 return False
3133 if self.keepRepoPath and not self.cloneDestination:
3134 sys.stderr.write("Must specify destination for --keep-path\n")
3135 sys.exit(1)
3137 depotPaths = args
3139 if not self.cloneDestination and len(depotPaths) > 1:
3140 self.cloneDestination = depotPaths[-1]
3141 depotPaths = depotPaths[:-1]
3143 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3144 for p in depotPaths:
3145 if not p.startswith("//"):
3146 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3147 return False
3149 if not self.cloneDestination:
3150 self.cloneDestination = self.defaultDestination(args)
3152 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3154 if not os.path.exists(self.cloneDestination):
3155 os.makedirs(self.cloneDestination)
3156 chdir(self.cloneDestination)
3158 init_cmd = [ "git", "init" ]
3159 if self.cloneBare:
3160 init_cmd.append("--bare")
3161 retcode = subprocess.call(init_cmd)
3162 if retcode:
3163 raise CalledProcessError(retcode, init_cmd)
3165 if not P4Sync.run(self, depotPaths):
3166 return False
3168 # create a master branch and check out a work tree
3169 if gitBranchExists(self.branch):
3170 system([ "git", "branch", "master", self.branch ])
3171 if not self.cloneBare:
3172 system([ "git", "checkout", "-f" ])
3173 else:
3174 print 'Not checking out any branch, use ' \
3175 '"git checkout -q -b master <branch>"'
3177 # auto-set this variable if invoked with --use-client-spec
3178 if self.useClientSpec_from_options:
3179 system("git config --bool git-p4.useclientspec true")
3181 return True
3183 class P4Branches(Command):
3184 def __init__(self):
3185 Command.__init__(self)
3186 self.options = [ ]
3187 self.description = ("Shows the git branches that hold imports and their "
3188 + "corresponding perforce depot paths")
3189 self.verbose = False
3191 def run(self, args):
3192 if originP4BranchesExist():
3193 createOrUpdateBranchesFromOrigin()
3195 cmdline = "git rev-parse --symbolic "
3196 cmdline += " --remotes"
3198 for line in read_pipe_lines(cmdline):
3199 line = line.strip()
3201 if not line.startswith('p4/') or line == "p4/HEAD":
3202 continue
3203 branch = line
3205 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3206 settings = extractSettingsGitLog(log)
3208 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3209 return True
3211 class HelpFormatter(optparse.IndentedHelpFormatter):
3212 def __init__(self):
3213 optparse.IndentedHelpFormatter.__init__(self)
3215 def format_description(self, description):
3216 if description:
3217 return description + "\n"
3218 else:
3219 return ""
3221 def printUsage(commands):
3222 print "usage: %s <command> [options]" % sys.argv[0]
3223 print ""
3224 print "valid commands: %s" % ", ".join(commands)
3225 print ""
3226 print "Try %s <command> --help for command specific help." % sys.argv[0]
3227 print ""
3229 commands = {
3230 "debug" : P4Debug,
3231 "submit" : P4Submit,
3232 "commit" : P4Submit,
3233 "sync" : P4Sync,
3234 "rebase" : P4Rebase,
3235 "clone" : P4Clone,
3236 "rollback" : P4RollBack,
3237 "branches" : P4Branches
3241 def main():
3242 if len(sys.argv[1:]) == 0:
3243 printUsage(commands.keys())
3244 sys.exit(2)
3246 cmdName = sys.argv[1]
3247 try:
3248 klass = commands[cmdName]
3249 cmd = klass()
3250 except KeyError:
3251 print "unknown command %s" % cmdName
3252 print ""
3253 printUsage(commands.keys())
3254 sys.exit(2)
3256 options = cmd.options
3257 cmd.gitdir = os.environ.get("GIT_DIR", None)
3259 args = sys.argv[2:]
3261 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3262 if cmd.needsGit:
3263 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3265 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3266 options,
3267 description = cmd.description,
3268 formatter = HelpFormatter())
3270 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3271 global verbose
3272 verbose = cmd.verbose
3273 if cmd.needsGit:
3274 if cmd.gitdir == None:
3275 cmd.gitdir = os.path.abspath(".git")
3276 if not isValidGitDir(cmd.gitdir):
3277 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3278 if os.path.exists(cmd.gitdir):
3279 cdup = read_pipe("git rev-parse --show-cdup").strip()
3280 if len(cdup) > 0:
3281 chdir(cdup);
3283 if not isValidGitDir(cmd.gitdir):
3284 if isValidGitDir(cmd.gitdir + "/.git"):
3285 cmd.gitdir += "/.git"
3286 else:
3287 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3289 os.environ["GIT_DIR"] = cmd.gitdir
3291 if not cmd.run(args):
3292 parser.print_help()
3293 sys.exit(2)
3296 if __name__ == '__main__':
3297 main()