repack: introduce repack.writeBitmaps config option
[git.git] / git-p4.py
blob06a3cc6122410efcccf0fd77f359ffc2e809aff9
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
26 try:
27 from subprocess import CalledProcessError
28 except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
41 verbose = False
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
46 def p4_build_cmd(cmd):
47 """Build a suitable p4 command line.
49 This consolidates building and returning a p4 command line into one
50 location. It means that hooking into the environment, or other configuration
51 can be done more easily.
52 """
53 real_cmd = ["p4"]
55 user = gitConfig("git-p4.user")
56 if len(user) > 0:
57 real_cmd += ["-u",user]
59 password = gitConfig("git-p4.password")
60 if len(password) > 0:
61 real_cmd += ["-P", password]
63 port = gitConfig("git-p4.port")
64 if len(port) > 0:
65 real_cmd += ["-p", port]
67 host = gitConfig("git-p4.host")
68 if len(host) > 0:
69 real_cmd += ["-H", host]
71 client = gitConfig("git-p4.client")
72 if len(client) > 0:
73 real_cmd += ["-c", client]
76 if isinstance(cmd,basestring):
77 real_cmd = ' '.join(real_cmd) + ' ' + cmd
78 else:
79 real_cmd += cmd
80 return real_cmd
82 def chdir(path, is_client_path=False):
83 """Do chdir to the given path, and set the PWD environment
84 variable for use by P4. It does not look at getcwd() output.
85 Since we're not using the shell, it is necessary to set the
86 PWD environment variable explicitly.
88 Normally, expand the path to force it to be absolute. This
89 addresses the use of relative path names inside P4 settings,
90 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
91 as given; it looks for .p4config using PWD.
93 If is_client_path, the path was handed to us directly by p4,
94 and may be a symbolic link. Do not call os.getcwd() in this
95 case, because it will cause p4 to think that PWD is not inside
96 the client path.
97 """
99 os.chdir(path)
100 if not is_client_path:
101 path = os.getcwd()
102 os.environ['PWD'] = path
104 def die(msg):
105 if verbose:
106 raise Exception(msg)
107 else:
108 sys.stderr.write(msg + "\n")
109 sys.exit(1)
111 def write_pipe(c, stdin):
112 if verbose:
113 sys.stderr.write('Writing pipe: %s\n' % str(c))
115 expand = isinstance(c,basestring)
116 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
117 pipe = p.stdin
118 val = pipe.write(stdin)
119 pipe.close()
120 if p.wait():
121 die('Command failed: %s' % str(c))
123 return val
125 def p4_write_pipe(c, stdin):
126 real_cmd = p4_build_cmd(c)
127 return write_pipe(real_cmd, stdin)
129 def read_pipe(c, ignore_error=False):
130 if verbose:
131 sys.stderr.write('Reading pipe: %s\n' % str(c))
133 expand = isinstance(c,basestring)
134 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
135 pipe = p.stdout
136 val = pipe.read()
137 if p.wait() and not ignore_error:
138 die('Command failed: %s' % str(c))
140 return val
142 def p4_read_pipe(c, ignore_error=False):
143 real_cmd = p4_build_cmd(c)
144 return read_pipe(real_cmd, ignore_error)
146 def read_pipe_lines(c):
147 if verbose:
148 sys.stderr.write('Reading pipe: %s\n' % str(c))
150 expand = isinstance(c, basestring)
151 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
152 pipe = p.stdout
153 val = pipe.readlines()
154 if pipe.close() or p.wait():
155 die('Command failed: %s' % str(c))
157 return val
159 def p4_read_pipe_lines(c):
160 """Specifically invoke p4 on the command supplied. """
161 real_cmd = p4_build_cmd(c)
162 return read_pipe_lines(real_cmd)
164 def p4_has_command(cmd):
165 """Ask p4 for help on this command. If it returns an error, the
166 command does not exist in this version of p4."""
167 real_cmd = p4_build_cmd(["help", cmd])
168 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
169 stderr=subprocess.PIPE)
170 p.communicate()
171 return p.returncode == 0
173 def p4_has_move_command():
174 """See if the move command exists, that it supports -k, and that
175 it has not been administratively disabled. The arguments
176 must be correct, but the filenames do not have to exist. Use
177 ones with wildcards so even if they exist, it will fail."""
179 if not p4_has_command("move"):
180 return False
181 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
182 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183 (out, err) = p.communicate()
184 # return code will be 1 in either case
185 if err.find("Invalid option") >= 0:
186 return False
187 if err.find("disabled") >= 0:
188 return False
189 # assume it failed because @... was invalid changelist
190 return True
192 def system(cmd):
193 expand = isinstance(cmd,basestring)
194 if verbose:
195 sys.stderr.write("executing %s\n" % str(cmd))
196 retcode = subprocess.call(cmd, shell=expand)
197 if retcode:
198 raise CalledProcessError(retcode, cmd)
200 def p4_system(cmd):
201 """Specifically invoke p4 as the system command. """
202 real_cmd = p4_build_cmd(cmd)
203 expand = isinstance(real_cmd, basestring)
204 retcode = subprocess.call(real_cmd, shell=expand)
205 if retcode:
206 raise CalledProcessError(retcode, real_cmd)
208 _p4_version_string = None
209 def p4_version_string():
210 """Read the version string, showing just the last line, which
211 hopefully is the interesting version bit.
213 $ p4 -V
214 Perforce - The Fast Software Configuration Management System.
215 Copyright 1995-2011 Perforce Software. All rights reserved.
216 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
218 global _p4_version_string
219 if not _p4_version_string:
220 a = p4_read_pipe_lines(["-V"])
221 _p4_version_string = a[-1].rstrip()
222 return _p4_version_string
224 def p4_integrate(src, dest):
225 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
227 def p4_sync(f, *options):
228 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
230 def p4_add(f):
231 # forcibly add file names with wildcards
232 if wildcard_present(f):
233 p4_system(["add", "-f", f])
234 else:
235 p4_system(["add", f])
237 def p4_delete(f):
238 p4_system(["delete", wildcard_encode(f)])
240 def p4_edit(f):
241 p4_system(["edit", wildcard_encode(f)])
243 def p4_revert(f):
244 p4_system(["revert", wildcard_encode(f)])
246 def p4_reopen(type, f):
247 p4_system(["reopen", "-t", type, wildcard_encode(f)])
249 def p4_move(src, dest):
250 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
252 def p4_describe(change):
253 """Make sure it returns a valid result by checking for
254 the presence of field "time". Return a dict of the
255 results."""
257 ds = p4CmdList(["describe", "-s", str(change)])
258 if len(ds) != 1:
259 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
261 d = ds[0]
263 if "p4ExitCode" in d:
264 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
265 str(d)))
266 if "code" in d:
267 if d["code"] == "error":
268 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
270 if "time" not in d:
271 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
273 return d
276 # Canonicalize the p4 type and return a tuple of the
277 # base type, plus any modifiers. See "p4 help filetypes"
278 # for a list and explanation.
280 def split_p4_type(p4type):
282 p4_filetypes_historical = {
283 "ctempobj": "binary+Sw",
284 "ctext": "text+C",
285 "cxtext": "text+Cx",
286 "ktext": "text+k",
287 "kxtext": "text+kx",
288 "ltext": "text+F",
289 "tempobj": "binary+FSw",
290 "ubinary": "binary+F",
291 "uresource": "resource+F",
292 "uxbinary": "binary+Fx",
293 "xbinary": "binary+x",
294 "xltext": "text+Fx",
295 "xtempobj": "binary+Swx",
296 "xtext": "text+x",
297 "xunicode": "unicode+x",
298 "xutf16": "utf16+x",
300 if p4type in p4_filetypes_historical:
301 p4type = p4_filetypes_historical[p4type]
302 mods = ""
303 s = p4type.split("+")
304 base = s[0]
305 mods = ""
306 if len(s) > 1:
307 mods = s[1]
308 return (base, mods)
311 # return the raw p4 type of a file (text, text+ko, etc)
313 def p4_type(file):
314 results = p4CmdList(["fstat", "-T", "headType", file])
315 return results[0]['headType']
318 # Given a type base and modifier, return a regexp matching
319 # the keywords that can be expanded in the file
321 def p4_keywords_regexp_for_type(base, type_mods):
322 if base in ("text", "unicode", "binary"):
323 kwords = None
324 if "ko" in type_mods:
325 kwords = 'Id|Header'
326 elif "k" in type_mods:
327 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
328 else:
329 return None
330 pattern = r"""
331 \$ # Starts with a dollar, followed by...
332 (%s) # one of the keywords, followed by...
333 (:[^$\n]+)? # possibly an old expansion, followed by...
334 \$ # another dollar
335 """ % kwords
336 return pattern
337 else:
338 return None
341 # Given a file, return a regexp matching the possible
342 # RCS keywords that will be expanded, or None for files
343 # with kw expansion turned off.
345 def p4_keywords_regexp_for_file(file):
346 if not os.path.exists(file):
347 return None
348 else:
349 (type_base, type_mods) = split_p4_type(p4_type(file))
350 return p4_keywords_regexp_for_type(type_base, type_mods)
352 def setP4ExecBit(file, mode):
353 # Reopens an already open file and changes the execute bit to match
354 # the execute bit setting in the passed in mode.
356 p4Type = "+x"
358 if not isModeExec(mode):
359 p4Type = getP4OpenedType(file)
360 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
361 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
362 if p4Type[-1] == "+":
363 p4Type = p4Type[0:-1]
365 p4_reopen(p4Type, file)
367 def getP4OpenedType(file):
368 # Returns the perforce file type for the given file.
370 result = p4_read_pipe(["opened", wildcard_encode(file)])
371 match = re.match(".*\((.+)\)\r?$", result)
372 if match:
373 return match.group(1)
374 else:
375 die("Could not determine file type for %s (result: '%s')" % (file, result))
377 # Return the set of all p4 labels
378 def getP4Labels(depotPaths):
379 labels = set()
380 if isinstance(depotPaths,basestring):
381 depotPaths = [depotPaths]
383 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
384 label = l['label']
385 labels.add(label)
387 return labels
389 # Return the set of all git tags
390 def getGitTags():
391 gitTags = set()
392 for line in read_pipe_lines(["git", "tag"]):
393 tag = line.strip()
394 gitTags.add(tag)
395 return gitTags
397 def diffTreePattern():
398 # This is a simple generator for the diff tree regex pattern. This could be
399 # a class variable if this and parseDiffTreeEntry were a part of a class.
400 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
401 while True:
402 yield pattern
404 def parseDiffTreeEntry(entry):
405 """Parses a single diff tree entry into its component elements.
407 See git-diff-tree(1) manpage for details about the format of the diff
408 output. This method returns a dictionary with the following elements:
410 src_mode - The mode of the source file
411 dst_mode - The mode of the destination file
412 src_sha1 - The sha1 for the source file
413 dst_sha1 - The sha1 fr the destination file
414 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
415 status_score - The score for the status (applicable for 'C' and 'R'
416 statuses). This is None if there is no score.
417 src - The path for the source file.
418 dst - The path for the destination file. This is only present for
419 copy or renames. If it is not present, this is None.
421 If the pattern is not matched, None is returned."""
423 match = diffTreePattern().next().match(entry)
424 if match:
425 return {
426 'src_mode': match.group(1),
427 'dst_mode': match.group(2),
428 'src_sha1': match.group(3),
429 'dst_sha1': match.group(4),
430 'status': match.group(5),
431 'status_score': match.group(6),
432 'src': match.group(7),
433 'dst': match.group(10)
435 return None
437 def isModeExec(mode):
438 # Returns True if the given git mode represents an executable file,
439 # otherwise False.
440 return mode[-3:] == "755"
442 def isModeExecChanged(src_mode, dst_mode):
443 return isModeExec(src_mode) != isModeExec(dst_mode)
445 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
447 if isinstance(cmd,basestring):
448 cmd = "-G " + cmd
449 expand = True
450 else:
451 cmd = ["-G"] + cmd
452 expand = False
454 cmd = p4_build_cmd(cmd)
455 if verbose:
456 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
458 # Use a temporary file to avoid deadlocks without
459 # subprocess.communicate(), which would put another copy
460 # of stdout into memory.
461 stdin_file = None
462 if stdin is not None:
463 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
464 if isinstance(stdin,basestring):
465 stdin_file.write(stdin)
466 else:
467 for i in stdin:
468 stdin_file.write(i + '\n')
469 stdin_file.flush()
470 stdin_file.seek(0)
472 p4 = subprocess.Popen(cmd,
473 shell=expand,
474 stdin=stdin_file,
475 stdout=subprocess.PIPE)
477 result = []
478 try:
479 while True:
480 entry = marshal.load(p4.stdout)
481 if cb is not None:
482 cb(entry)
483 else:
484 result.append(entry)
485 except EOFError:
486 pass
487 exitCode = p4.wait()
488 if exitCode != 0:
489 entry = {}
490 entry["p4ExitCode"] = exitCode
491 result.append(entry)
493 return result
495 def p4Cmd(cmd):
496 list = p4CmdList(cmd)
497 result = {}
498 for entry in list:
499 result.update(entry)
500 return result;
502 def p4Where(depotPath):
503 if not depotPath.endswith("/"):
504 depotPath += "/"
505 depotPath = depotPath + "..."
506 outputList = p4CmdList(["where", depotPath])
507 output = None
508 for entry in outputList:
509 if "depotFile" in entry:
510 if entry["depotFile"] == depotPath:
511 output = entry
512 break
513 elif "data" in entry:
514 data = entry.get("data")
515 space = data.find(" ")
516 if data[:space] == depotPath:
517 output = entry
518 break
519 if output == None:
520 return ""
521 if output["code"] == "error":
522 return ""
523 clientPath = ""
524 if "path" in output:
525 clientPath = output.get("path")
526 elif "data" in output:
527 data = output.get("data")
528 lastSpace = data.rfind(" ")
529 clientPath = data[lastSpace + 1:]
531 if clientPath.endswith("..."):
532 clientPath = clientPath[:-3]
533 return clientPath
535 def currentGitBranch():
536 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
538 def isValidGitDir(path):
539 if (os.path.exists(path + "/HEAD")
540 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
541 return True;
542 return False
544 def parseRevision(ref):
545 return read_pipe("git rev-parse %s" % ref).strip()
547 def branchExists(ref):
548 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
549 ignore_error=True)
550 return len(rev) > 0
552 def extractLogMessageFromGitCommit(commit):
553 logMessage = ""
555 ## fixme: title is first line of commit, not 1st paragraph.
556 foundTitle = False
557 for log in read_pipe_lines("git cat-file commit %s" % commit):
558 if not foundTitle:
559 if len(log) == 1:
560 foundTitle = True
561 continue
563 logMessage += log
564 return logMessage
566 def extractSettingsGitLog(log):
567 values = {}
568 for line in log.split("\n"):
569 line = line.strip()
570 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
571 if not m:
572 continue
574 assignments = m.group(1).split (':')
575 for a in assignments:
576 vals = a.split ('=')
577 key = vals[0].strip()
578 val = ('='.join (vals[1:])).strip()
579 if val.endswith ('\"') and val.startswith('"'):
580 val = val[1:-1]
582 values[key] = val
584 paths = values.get("depot-paths")
585 if not paths:
586 paths = values.get("depot-path")
587 if paths:
588 values['depot-paths'] = paths.split(',')
589 return values
591 def gitBranchExists(branch):
592 proc = subprocess.Popen(["git", "rev-parse", branch],
593 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
594 return proc.wait() == 0;
596 _gitConfig = {}
598 def gitConfig(key):
599 if not _gitConfig.has_key(key):
600 cmd = [ "git", "config", key ]
601 s = read_pipe(cmd, ignore_error=True)
602 _gitConfig[key] = s.strip()
603 return _gitConfig[key]
605 def gitConfigBool(key):
606 """Return a bool, using git config --bool. It is True only if the
607 variable is set to true, and False if set to false or not present
608 in the config."""
610 if not _gitConfig.has_key(key):
611 cmd = [ "git", "config", "--bool", key ]
612 s = read_pipe(cmd, ignore_error=True)
613 v = s.strip()
614 _gitConfig[key] = v == "true"
615 return _gitConfig[key]
617 def gitConfigList(key):
618 if not _gitConfig.has_key(key):
619 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
620 _gitConfig[key] = s.strip().split(os.linesep)
621 return _gitConfig[key]
623 def p4BranchesInGit(branchesAreInRemotes=True):
624 """Find all the branches whose names start with "p4/", looking
625 in remotes or heads as specified by the argument. Return
626 a dictionary of { branch: revision } for each one found.
627 The branch names are the short names, without any
628 "p4/" prefix."""
630 branches = {}
632 cmdline = "git rev-parse --symbolic "
633 if branchesAreInRemotes:
634 cmdline += "--remotes"
635 else:
636 cmdline += "--branches"
638 for line in read_pipe_lines(cmdline):
639 line = line.strip()
641 # only import to p4/
642 if not line.startswith('p4/'):
643 continue
644 # special symbolic ref to p4/master
645 if line == "p4/HEAD":
646 continue
648 # strip off p4/ prefix
649 branch = line[len("p4/"):]
651 branches[branch] = parseRevision(line)
653 return branches
655 def branch_exists(branch):
656 """Make sure that the given ref name really exists."""
658 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
659 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
660 out, _ = p.communicate()
661 if p.returncode:
662 return False
663 # expect exactly one line of output: the branch name
664 return out.rstrip() == branch
666 def findUpstreamBranchPoint(head = "HEAD"):
667 branches = p4BranchesInGit()
668 # map from depot-path to branch name
669 branchByDepotPath = {}
670 for branch in branches.keys():
671 tip = branches[branch]
672 log = extractLogMessageFromGitCommit(tip)
673 settings = extractSettingsGitLog(log)
674 if settings.has_key("depot-paths"):
675 paths = ",".join(settings["depot-paths"])
676 branchByDepotPath[paths] = "remotes/p4/" + branch
678 settings = None
679 parent = 0
680 while parent < 65535:
681 commit = head + "~%s" % parent
682 log = extractLogMessageFromGitCommit(commit)
683 settings = extractSettingsGitLog(log)
684 if settings.has_key("depot-paths"):
685 paths = ",".join(settings["depot-paths"])
686 if branchByDepotPath.has_key(paths):
687 return [branchByDepotPath[paths], settings]
689 parent = parent + 1
691 return ["", settings]
693 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
694 if not silent:
695 print ("Creating/updating branch(es) in %s based on origin branch(es)"
696 % localRefPrefix)
698 originPrefix = "origin/p4/"
700 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
701 line = line.strip()
702 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
703 continue
705 headName = line[len(originPrefix):]
706 remoteHead = localRefPrefix + headName
707 originHead = line
709 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
710 if (not original.has_key('depot-paths')
711 or not original.has_key('change')):
712 continue
714 update = False
715 if not gitBranchExists(remoteHead):
716 if verbose:
717 print "creating %s" % remoteHead
718 update = True
719 else:
720 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
721 if settings.has_key('change') > 0:
722 if settings['depot-paths'] == original['depot-paths']:
723 originP4Change = int(original['change'])
724 p4Change = int(settings['change'])
725 if originP4Change > p4Change:
726 print ("%s (%s) is newer than %s (%s). "
727 "Updating p4 branch from origin."
728 % (originHead, originP4Change,
729 remoteHead, p4Change))
730 update = True
731 else:
732 print ("Ignoring: %s was imported from %s while "
733 "%s was imported from %s"
734 % (originHead, ','.join(original['depot-paths']),
735 remoteHead, ','.join(settings['depot-paths'])))
737 if update:
738 system("git update-ref %s %s" % (remoteHead, originHead))
740 def originP4BranchesExist():
741 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
743 def p4ChangesForPaths(depotPaths, changeRange):
744 assert depotPaths
745 cmd = ['changes']
746 for p in depotPaths:
747 cmd += ["%s...%s" % (p, changeRange)]
748 output = p4_read_pipe_lines(cmd)
750 changes = {}
751 for line in output:
752 changeNum = int(line.split(" ")[1])
753 changes[changeNum] = True
755 changelist = changes.keys()
756 changelist.sort()
757 return changelist
759 def p4PathStartsWith(path, prefix):
760 # This method tries to remedy a potential mixed-case issue:
762 # If UserA adds //depot/DirA/file1
763 # and UserB adds //depot/dira/file2
765 # we may or may not have a problem. If you have core.ignorecase=true,
766 # we treat DirA and dira as the same directory
767 if gitConfigBool("core.ignorecase"):
768 return path.lower().startswith(prefix.lower())
769 return path.startswith(prefix)
771 def getClientSpec():
772 """Look at the p4 client spec, create a View() object that contains
773 all the mappings, and return it."""
775 specList = p4CmdList("client -o")
776 if len(specList) != 1:
777 die('Output from "client -o" is %d lines, expecting 1' %
778 len(specList))
780 # dictionary of all client parameters
781 entry = specList[0]
783 # the //client/ name
784 client_name = entry["Client"]
786 # just the keys that start with "View"
787 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
789 # hold this new View
790 view = View(client_name)
792 # append the lines, in order, to the view
793 for view_num in range(len(view_keys)):
794 k = "View%d" % view_num
795 if k not in view_keys:
796 die("Expected view key %s missing" % k)
797 view.append(entry[k])
799 return view
801 def getClientRoot():
802 """Grab the client directory."""
804 output = p4CmdList("client -o")
805 if len(output) != 1:
806 die('Output from "client -o" is %d lines, expecting 1' % len(output))
808 entry = output[0]
809 if "Root" not in entry:
810 die('Client has no "Root"')
812 return entry["Root"]
815 # P4 wildcards are not allowed in filenames. P4 complains
816 # if you simply add them, but you can force it with "-f", in
817 # which case it translates them into %xx encoding internally.
819 def wildcard_decode(path):
820 # Search for and fix just these four characters. Do % last so
821 # that fixing it does not inadvertently create new %-escapes.
822 # Cannot have * in a filename in windows; untested as to
823 # what p4 would do in such a case.
824 if not platform.system() == "Windows":
825 path = path.replace("%2A", "*")
826 path = path.replace("%23", "#") \
827 .replace("%40", "@") \
828 .replace("%25", "%")
829 return path
831 def wildcard_encode(path):
832 # do % first to avoid double-encoding the %s introduced here
833 path = path.replace("%", "%25") \
834 .replace("*", "%2A") \
835 .replace("#", "%23") \
836 .replace("@", "%40")
837 return path
839 def wildcard_present(path):
840 m = re.search("[*#@%]", path)
841 return m is not None
843 class Command:
844 def __init__(self):
845 self.usage = "usage: %prog [options]"
846 self.needsGit = True
847 self.verbose = False
849 class P4UserMap:
850 def __init__(self):
851 self.userMapFromPerforceServer = False
852 self.myP4UserId = None
854 def p4UserId(self):
855 if self.myP4UserId:
856 return self.myP4UserId
858 results = p4CmdList("user -o")
859 for r in results:
860 if r.has_key('User'):
861 self.myP4UserId = r['User']
862 return r['User']
863 die("Could not find your p4 user id")
865 def p4UserIsMe(self, p4User):
866 # return True if the given p4 user is actually me
867 me = self.p4UserId()
868 if not p4User or p4User != me:
869 return False
870 else:
871 return True
873 def getUserCacheFilename(self):
874 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
875 return home + "/.gitp4-usercache.txt"
877 def getUserMapFromPerforceServer(self):
878 if self.userMapFromPerforceServer:
879 return
880 self.users = {}
881 self.emails = {}
883 for output in p4CmdList("users"):
884 if not output.has_key("User"):
885 continue
886 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
887 self.emails[output["Email"]] = output["User"]
890 s = ''
891 for (key, val) in self.users.items():
892 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
894 open(self.getUserCacheFilename(), "wb").write(s)
895 self.userMapFromPerforceServer = True
897 def loadUserMapFromCache(self):
898 self.users = {}
899 self.userMapFromPerforceServer = False
900 try:
901 cache = open(self.getUserCacheFilename(), "rb")
902 lines = cache.readlines()
903 cache.close()
904 for line in lines:
905 entry = line.strip().split("\t")
906 self.users[entry[0]] = entry[1]
907 except IOError:
908 self.getUserMapFromPerforceServer()
910 class P4Debug(Command):
911 def __init__(self):
912 Command.__init__(self)
913 self.options = []
914 self.description = "A tool to debug the output of p4 -G."
915 self.needsGit = False
917 def run(self, args):
918 j = 0
919 for output in p4CmdList(args):
920 print 'Element: %d' % j
921 j += 1
922 print output
923 return True
925 class P4RollBack(Command):
926 def __init__(self):
927 Command.__init__(self)
928 self.options = [
929 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
931 self.description = "A tool to debug the multi-branch import. Don't use :)"
932 self.rollbackLocalBranches = False
934 def run(self, args):
935 if len(args) != 1:
936 return False
937 maxChange = int(args[0])
939 if "p4ExitCode" in p4Cmd("changes -m 1"):
940 die("Problems executing p4");
942 if self.rollbackLocalBranches:
943 refPrefix = "refs/heads/"
944 lines = read_pipe_lines("git rev-parse --symbolic --branches")
945 else:
946 refPrefix = "refs/remotes/"
947 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
949 for line in lines:
950 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
951 line = line.strip()
952 ref = refPrefix + line
953 log = extractLogMessageFromGitCommit(ref)
954 settings = extractSettingsGitLog(log)
956 depotPaths = settings['depot-paths']
957 change = settings['change']
959 changed = False
961 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
962 for p in depotPaths]))) == 0:
963 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
964 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
965 continue
967 while change and int(change) > maxChange:
968 changed = True
969 if self.verbose:
970 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
971 system("git update-ref %s \"%s^\"" % (ref, ref))
972 log = extractLogMessageFromGitCommit(ref)
973 settings = extractSettingsGitLog(log)
976 depotPaths = settings['depot-paths']
977 change = settings['change']
979 if changed:
980 print "%s rewound to %s" % (ref, change)
982 return True
984 class P4Submit(Command, P4UserMap):
986 conflict_behavior_choices = ("ask", "skip", "quit")
988 def __init__(self):
989 Command.__init__(self)
990 P4UserMap.__init__(self)
991 self.options = [
992 optparse.make_option("--origin", dest="origin"),
993 optparse.make_option("-M", dest="detectRenames", action="store_true"),
994 # preserve the user, requires relevant p4 permissions
995 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
996 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
997 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
998 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
999 optparse.make_option("--conflict", dest="conflict_behavior",
1000 choices=self.conflict_behavior_choices),
1001 optparse.make_option("--branch", dest="branch"),
1003 self.description = "Submit changes from git to the perforce depot."
1004 self.usage += " [name of git branch to submit into perforce depot]"
1005 self.origin = ""
1006 self.detectRenames = False
1007 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1008 self.dry_run = False
1009 self.prepare_p4_only = False
1010 self.conflict_behavior = None
1011 self.isWindows = (platform.system() == "Windows")
1012 self.exportLabels = False
1013 self.p4HasMoveCommand = p4_has_move_command()
1014 self.branch = None
1016 def check(self):
1017 if len(p4CmdList("opened ...")) > 0:
1018 die("You have files opened with perforce! Close them before starting the sync.")
1020 def separate_jobs_from_description(self, message):
1021 """Extract and return a possible Jobs field in the commit
1022 message. It goes into a separate section in the p4 change
1023 specification.
1025 A jobs line starts with "Jobs:" and looks like a new field
1026 in a form. Values are white-space separated on the same
1027 line or on following lines that start with a tab.
1029 This does not parse and extract the full git commit message
1030 like a p4 form. It just sees the Jobs: line as a marker
1031 to pass everything from then on directly into the p4 form,
1032 but outside the description section.
1034 Return a tuple (stripped log message, jobs string)."""
1036 m = re.search(r'^Jobs:', message, re.MULTILINE)
1037 if m is None:
1038 return (message, None)
1040 jobtext = message[m.start():]
1041 stripped_message = message[:m.start()].rstrip()
1042 return (stripped_message, jobtext)
1044 def prepareLogMessage(self, template, message, jobs):
1045 """Edits the template returned from "p4 change -o" to insert
1046 the message in the Description field, and the jobs text in
1047 the Jobs field."""
1048 result = ""
1050 inDescriptionSection = False
1052 for line in template.split("\n"):
1053 if line.startswith("#"):
1054 result += line + "\n"
1055 continue
1057 if inDescriptionSection:
1058 if line.startswith("Files:") or line.startswith("Jobs:"):
1059 inDescriptionSection = False
1060 # insert Jobs section
1061 if jobs:
1062 result += jobs + "\n"
1063 else:
1064 continue
1065 else:
1066 if line.startswith("Description:"):
1067 inDescriptionSection = True
1068 line += "\n"
1069 for messageLine in message.split("\n"):
1070 line += "\t" + messageLine + "\n"
1072 result += line + "\n"
1074 return result
1076 def patchRCSKeywords(self, file, pattern):
1077 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1078 (handle, outFileName) = tempfile.mkstemp(dir='.')
1079 try:
1080 outFile = os.fdopen(handle, "w+")
1081 inFile = open(file, "r")
1082 regexp = re.compile(pattern, re.VERBOSE)
1083 for line in inFile.readlines():
1084 line = regexp.sub(r'$\1$', line)
1085 outFile.write(line)
1086 inFile.close()
1087 outFile.close()
1088 # Forcibly overwrite the original file
1089 os.unlink(file)
1090 shutil.move(outFileName, file)
1091 except:
1092 # cleanup our temporary file
1093 os.unlink(outFileName)
1094 print "Failed to strip RCS keywords in %s" % file
1095 raise
1097 print "Patched up RCS keywords in %s" % file
1099 def p4UserForCommit(self,id):
1100 # Return the tuple (perforce user,git email) for a given git commit id
1101 self.getUserMapFromPerforceServer()
1102 gitEmail = read_pipe(["git", "log", "--max-count=1",
1103 "--format=%ae", id])
1104 gitEmail = gitEmail.strip()
1105 if not self.emails.has_key(gitEmail):
1106 return (None,gitEmail)
1107 else:
1108 return (self.emails[gitEmail],gitEmail)
1110 def checkValidP4Users(self,commits):
1111 # check if any git authors cannot be mapped to p4 users
1112 for id in commits:
1113 (user,email) = self.p4UserForCommit(id)
1114 if not user:
1115 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1116 if gitConfigBool("git-p4.allowMissingP4Users"):
1117 print "%s" % msg
1118 else:
1119 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1121 def lastP4Changelist(self):
1122 # Get back the last changelist number submitted in this client spec. This
1123 # then gets used to patch up the username in the change. If the same
1124 # client spec is being used by multiple processes then this might go
1125 # wrong.
1126 results = p4CmdList("client -o") # find the current client
1127 client = None
1128 for r in results:
1129 if r.has_key('Client'):
1130 client = r['Client']
1131 break
1132 if not client:
1133 die("could not get client spec")
1134 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1135 for r in results:
1136 if r.has_key('change'):
1137 return r['change']
1138 die("Could not get changelist number for last submit - cannot patch up user details")
1140 def modifyChangelistUser(self, changelist, newUser):
1141 # fixup the user field of a changelist after it has been submitted.
1142 changes = p4CmdList("change -o %s" % changelist)
1143 if len(changes) != 1:
1144 die("Bad output from p4 change modifying %s to user %s" %
1145 (changelist, newUser))
1147 c = changes[0]
1148 if c['User'] == newUser: return # nothing to do
1149 c['User'] = newUser
1150 input = marshal.dumps(c)
1152 result = p4CmdList("change -f -i", stdin=input)
1153 for r in result:
1154 if r.has_key('code'):
1155 if r['code'] == 'error':
1156 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1157 if r.has_key('data'):
1158 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1159 return
1160 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1162 def canChangeChangelists(self):
1163 # check to see if we have p4 admin or super-user permissions, either of
1164 # which are required to modify changelists.
1165 results = p4CmdList(["protects", self.depotPath])
1166 for r in results:
1167 if r.has_key('perm'):
1168 if r['perm'] == 'admin':
1169 return 1
1170 if r['perm'] == 'super':
1171 return 1
1172 return 0
1174 def prepareSubmitTemplate(self):
1175 """Run "p4 change -o" to grab a change specification template.
1176 This does not use "p4 -G", as it is nice to keep the submission
1177 template in original order, since a human might edit it.
1179 Remove lines in the Files section that show changes to files
1180 outside the depot path we're committing into."""
1182 template = ""
1183 inFilesSection = False
1184 for line in p4_read_pipe_lines(['change', '-o']):
1185 if line.endswith("\r\n"):
1186 line = line[:-2] + "\n"
1187 if inFilesSection:
1188 if line.startswith("\t"):
1189 # path starts and ends with a tab
1190 path = line[1:]
1191 lastTab = path.rfind("\t")
1192 if lastTab != -1:
1193 path = path[:lastTab]
1194 if not p4PathStartsWith(path, self.depotPath):
1195 continue
1196 else:
1197 inFilesSection = False
1198 else:
1199 if line.startswith("Files:"):
1200 inFilesSection = True
1202 template += line
1204 return template
1206 def edit_template(self, template_file):
1207 """Invoke the editor to let the user change the submission
1208 message. Return true if okay to continue with the submit."""
1210 # if configured to skip the editing part, just submit
1211 if gitConfigBool("git-p4.skipSubmitEdit"):
1212 return True
1214 # look at the modification time, to check later if the user saved
1215 # the file
1216 mtime = os.stat(template_file).st_mtime
1218 # invoke the editor
1219 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1220 editor = os.environ.get("P4EDITOR")
1221 else:
1222 editor = read_pipe("git var GIT_EDITOR").strip()
1223 system(editor + " " + template_file)
1225 # If the file was not saved, prompt to see if this patch should
1226 # be skipped. But skip this verification step if configured so.
1227 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1228 return True
1230 # modification time updated means user saved the file
1231 if os.stat(template_file).st_mtime > mtime:
1232 return True
1234 while True:
1235 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1236 if response == 'y':
1237 return True
1238 if response == 'n':
1239 return False
1241 def applyCommit(self, id):
1242 """Apply one commit, return True if it succeeded."""
1244 print "Applying", read_pipe(["git", "show", "-s",
1245 "--format=format:%h %s", id])
1247 (p4User, gitEmail) = self.p4UserForCommit(id)
1249 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1250 filesToAdd = set()
1251 filesToDelete = set()
1252 editedFiles = set()
1253 pureRenameCopy = set()
1254 filesToChangeExecBit = {}
1256 for line in diff:
1257 diff = parseDiffTreeEntry(line)
1258 modifier = diff['status']
1259 path = diff['src']
1260 if modifier == "M":
1261 p4_edit(path)
1262 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1263 filesToChangeExecBit[path] = diff['dst_mode']
1264 editedFiles.add(path)
1265 elif modifier == "A":
1266 filesToAdd.add(path)
1267 filesToChangeExecBit[path] = diff['dst_mode']
1268 if path in filesToDelete:
1269 filesToDelete.remove(path)
1270 elif modifier == "D":
1271 filesToDelete.add(path)
1272 if path in filesToAdd:
1273 filesToAdd.remove(path)
1274 elif modifier == "C":
1275 src, dest = diff['src'], diff['dst']
1276 p4_integrate(src, dest)
1277 pureRenameCopy.add(dest)
1278 if diff['src_sha1'] != diff['dst_sha1']:
1279 p4_edit(dest)
1280 pureRenameCopy.discard(dest)
1281 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1282 p4_edit(dest)
1283 pureRenameCopy.discard(dest)
1284 filesToChangeExecBit[dest] = diff['dst_mode']
1285 if self.isWindows:
1286 # turn off read-only attribute
1287 os.chmod(dest, stat.S_IWRITE)
1288 os.unlink(dest)
1289 editedFiles.add(dest)
1290 elif modifier == "R":
1291 src, dest = diff['src'], diff['dst']
1292 if self.p4HasMoveCommand:
1293 p4_edit(src) # src must be open before move
1294 p4_move(src, dest) # opens for (move/delete, move/add)
1295 else:
1296 p4_integrate(src, dest)
1297 if diff['src_sha1'] != diff['dst_sha1']:
1298 p4_edit(dest)
1299 else:
1300 pureRenameCopy.add(dest)
1301 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1302 if not self.p4HasMoveCommand:
1303 p4_edit(dest) # with move: already open, writable
1304 filesToChangeExecBit[dest] = diff['dst_mode']
1305 if not self.p4HasMoveCommand:
1306 if self.isWindows:
1307 os.chmod(dest, stat.S_IWRITE)
1308 os.unlink(dest)
1309 filesToDelete.add(src)
1310 editedFiles.add(dest)
1311 else:
1312 die("unknown modifier %s for %s" % (modifier, path))
1314 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1315 patchcmd = diffcmd + " | git apply "
1316 tryPatchCmd = patchcmd + "--check -"
1317 applyPatchCmd = patchcmd + "--check --apply -"
1318 patch_succeeded = True
1320 if os.system(tryPatchCmd) != 0:
1321 fixed_rcs_keywords = False
1322 patch_succeeded = False
1323 print "Unfortunately applying the change failed!"
1325 # Patch failed, maybe it's just RCS keyword woes. Look through
1326 # the patch to see if that's possible.
1327 if gitConfigBool("git-p4.attemptRCSCleanup"):
1328 file = None
1329 pattern = None
1330 kwfiles = {}
1331 for file in editedFiles | filesToDelete:
1332 # did this file's delta contain RCS keywords?
1333 pattern = p4_keywords_regexp_for_file(file)
1335 if pattern:
1336 # this file is a possibility...look for RCS keywords.
1337 regexp = re.compile(pattern, re.VERBOSE)
1338 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1339 if regexp.search(line):
1340 if verbose:
1341 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1342 kwfiles[file] = pattern
1343 break
1345 for file in kwfiles:
1346 if verbose:
1347 print "zapping %s with %s" % (line,pattern)
1348 # File is being deleted, so not open in p4. Must
1349 # disable the read-only bit on windows.
1350 if self.isWindows and file not in editedFiles:
1351 os.chmod(file, stat.S_IWRITE)
1352 self.patchRCSKeywords(file, kwfiles[file])
1353 fixed_rcs_keywords = True
1355 if fixed_rcs_keywords:
1356 print "Retrying the patch with RCS keywords cleaned up"
1357 if os.system(tryPatchCmd) == 0:
1358 patch_succeeded = True
1360 if not patch_succeeded:
1361 for f in editedFiles:
1362 p4_revert(f)
1363 return False
1366 # Apply the patch for real, and do add/delete/+x handling.
1368 system(applyPatchCmd)
1370 for f in filesToAdd:
1371 p4_add(f)
1372 for f in filesToDelete:
1373 p4_revert(f)
1374 p4_delete(f)
1376 # Set/clear executable bits
1377 for f in filesToChangeExecBit.keys():
1378 mode = filesToChangeExecBit[f]
1379 setP4ExecBit(f, mode)
1382 # Build p4 change description, starting with the contents
1383 # of the git commit message.
1385 logMessage = extractLogMessageFromGitCommit(id)
1386 logMessage = logMessage.strip()
1387 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1389 template = self.prepareSubmitTemplate()
1390 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1392 if self.preserveUser:
1393 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1395 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1396 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1397 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1398 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1400 separatorLine = "######## everything below this line is just the diff #######\n"
1402 # diff
1403 if os.environ.has_key("P4DIFF"):
1404 del(os.environ["P4DIFF"])
1405 diff = ""
1406 for editedFile in editedFiles:
1407 diff += p4_read_pipe(['diff', '-du',
1408 wildcard_encode(editedFile)])
1410 # new file diff
1411 newdiff = ""
1412 for newFile in filesToAdd:
1413 newdiff += "==== new file ====\n"
1414 newdiff += "--- /dev/null\n"
1415 newdiff += "+++ %s\n" % newFile
1416 f = open(newFile, "r")
1417 for line in f.readlines():
1418 newdiff += "+" + line
1419 f.close()
1421 # change description file: submitTemplate, separatorLine, diff, newdiff
1422 (handle, fileName) = tempfile.mkstemp()
1423 tmpFile = os.fdopen(handle, "w+")
1424 if self.isWindows:
1425 submitTemplate = submitTemplate.replace("\n", "\r\n")
1426 separatorLine = separatorLine.replace("\n", "\r\n")
1427 newdiff = newdiff.replace("\n", "\r\n")
1428 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1429 tmpFile.close()
1431 if self.prepare_p4_only:
1433 # Leave the p4 tree prepared, and the submit template around
1434 # and let the user decide what to do next
1436 print
1437 print "P4 workspace prepared for submission."
1438 print "To submit or revert, go to client workspace"
1439 print " " + self.clientPath
1440 print
1441 print "To submit, use \"p4 submit\" to write a new description,"
1442 print "or \"p4 submit -i %s\" to use the one prepared by" \
1443 " \"git p4\"." % fileName
1444 print "You can delete the file \"%s\" when finished." % fileName
1446 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1447 print "To preserve change ownership by user %s, you must\n" \
1448 "do \"p4 change -f <change>\" after submitting and\n" \
1449 "edit the User field."
1450 if pureRenameCopy:
1451 print "After submitting, renamed files must be re-synced."
1452 print "Invoke \"p4 sync -f\" on each of these files:"
1453 for f in pureRenameCopy:
1454 print " " + f
1456 print
1457 print "To revert the changes, use \"p4 revert ...\", and delete"
1458 print "the submit template file \"%s\"" % fileName
1459 if filesToAdd:
1460 print "Since the commit adds new files, they must be deleted:"
1461 for f in filesToAdd:
1462 print " " + f
1463 print
1464 return True
1467 # Let the user edit the change description, then submit it.
1469 if self.edit_template(fileName):
1470 # read the edited message and submit
1471 ret = True
1472 tmpFile = open(fileName, "rb")
1473 message = tmpFile.read()
1474 tmpFile.close()
1475 submitTemplate = message[:message.index(separatorLine)]
1476 if self.isWindows:
1477 submitTemplate = submitTemplate.replace("\r\n", "\n")
1478 p4_write_pipe(['submit', '-i'], submitTemplate)
1480 if self.preserveUser:
1481 if p4User:
1482 # Get last changelist number. Cannot easily get it from
1483 # the submit command output as the output is
1484 # unmarshalled.
1485 changelist = self.lastP4Changelist()
1486 self.modifyChangelistUser(changelist, p4User)
1488 # The rename/copy happened by applying a patch that created a
1489 # new file. This leaves it writable, which confuses p4.
1490 for f in pureRenameCopy:
1491 p4_sync(f, "-f")
1493 else:
1494 # skip this patch
1495 ret = False
1496 print "Submission cancelled, undoing p4 changes."
1497 for f in editedFiles:
1498 p4_revert(f)
1499 for f in filesToAdd:
1500 p4_revert(f)
1501 os.remove(f)
1502 for f in filesToDelete:
1503 p4_revert(f)
1505 os.remove(fileName)
1506 return ret
1508 # Export git tags as p4 labels. Create a p4 label and then tag
1509 # with that.
1510 def exportGitTags(self, gitTags):
1511 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1512 if len(validLabelRegexp) == 0:
1513 validLabelRegexp = defaultLabelRegexp
1514 m = re.compile(validLabelRegexp)
1516 for name in gitTags:
1518 if not m.match(name):
1519 if verbose:
1520 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1521 continue
1523 # Get the p4 commit this corresponds to
1524 logMessage = extractLogMessageFromGitCommit(name)
1525 values = extractSettingsGitLog(logMessage)
1527 if not values.has_key('change'):
1528 # a tag pointing to something not sent to p4; ignore
1529 if verbose:
1530 print "git tag %s does not give a p4 commit" % name
1531 continue
1532 else:
1533 changelist = values['change']
1535 # Get the tag details.
1536 inHeader = True
1537 isAnnotated = False
1538 body = []
1539 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1540 l = l.strip()
1541 if inHeader:
1542 if re.match(r'tag\s+', l):
1543 isAnnotated = True
1544 elif re.match(r'\s*$', l):
1545 inHeader = False
1546 continue
1547 else:
1548 body.append(l)
1550 if not isAnnotated:
1551 body = ["lightweight tag imported by git p4\n"]
1553 # Create the label - use the same view as the client spec we are using
1554 clientSpec = getClientSpec()
1556 labelTemplate = "Label: %s\n" % name
1557 labelTemplate += "Description:\n"
1558 for b in body:
1559 labelTemplate += "\t" + b + "\n"
1560 labelTemplate += "View:\n"
1561 for depot_side in clientSpec.mappings:
1562 labelTemplate += "\t%s\n" % depot_side
1564 if self.dry_run:
1565 print "Would create p4 label %s for tag" % name
1566 elif self.prepare_p4_only:
1567 print "Not creating p4 label %s for tag due to option" \
1568 " --prepare-p4-only" % name
1569 else:
1570 p4_write_pipe(["label", "-i"], labelTemplate)
1572 # Use the label
1573 p4_system(["tag", "-l", name] +
1574 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1576 if verbose:
1577 print "created p4 label for tag %s" % name
1579 def run(self, args):
1580 if len(args) == 0:
1581 self.master = currentGitBranch()
1582 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1583 die("Detecting current git branch failed!")
1584 elif len(args) == 1:
1585 self.master = args[0]
1586 if not branchExists(self.master):
1587 die("Branch %s does not exist" % self.master)
1588 else:
1589 return False
1591 allowSubmit = gitConfig("git-p4.allowSubmit")
1592 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1593 die("%s is not in git-p4.allowSubmit" % self.master)
1595 [upstream, settings] = findUpstreamBranchPoint()
1596 self.depotPath = settings['depot-paths'][0]
1597 if len(self.origin) == 0:
1598 self.origin = upstream
1600 if self.preserveUser:
1601 if not self.canChangeChangelists():
1602 die("Cannot preserve user names without p4 super-user or admin permissions")
1604 # if not set from the command line, try the config file
1605 if self.conflict_behavior is None:
1606 val = gitConfig("git-p4.conflict")
1607 if val:
1608 if val not in self.conflict_behavior_choices:
1609 die("Invalid value '%s' for config git-p4.conflict" % val)
1610 else:
1611 val = "ask"
1612 self.conflict_behavior = val
1614 if self.verbose:
1615 print "Origin branch is " + self.origin
1617 if len(self.depotPath) == 0:
1618 print "Internal error: cannot locate perforce depot path from existing branches"
1619 sys.exit(128)
1621 self.useClientSpec = False
1622 if gitConfigBool("git-p4.useclientspec"):
1623 self.useClientSpec = True
1624 if self.useClientSpec:
1625 self.clientSpecDirs = getClientSpec()
1627 if self.useClientSpec:
1628 # all files are relative to the client spec
1629 self.clientPath = getClientRoot()
1630 else:
1631 self.clientPath = p4Where(self.depotPath)
1633 if self.clientPath == "":
1634 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1636 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1637 self.oldWorkingDirectory = os.getcwd()
1639 # ensure the clientPath exists
1640 new_client_dir = False
1641 if not os.path.exists(self.clientPath):
1642 new_client_dir = True
1643 os.makedirs(self.clientPath)
1645 chdir(self.clientPath, is_client_path=True)
1646 if self.dry_run:
1647 print "Would synchronize p4 checkout in %s" % self.clientPath
1648 else:
1649 print "Synchronizing p4 checkout..."
1650 if new_client_dir:
1651 # old one was destroyed, and maybe nobody told p4
1652 p4_sync("...", "-f")
1653 else:
1654 p4_sync("...")
1655 self.check()
1657 commits = []
1658 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1659 commits.append(line.strip())
1660 commits.reverse()
1662 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1663 self.checkAuthorship = False
1664 else:
1665 self.checkAuthorship = True
1667 if self.preserveUser:
1668 self.checkValidP4Users(commits)
1671 # Build up a set of options to be passed to diff when
1672 # submitting each commit to p4.
1674 if self.detectRenames:
1675 # command-line -M arg
1676 self.diffOpts = "-M"
1677 else:
1678 # If not explicitly set check the config variable
1679 detectRenames = gitConfig("git-p4.detectRenames")
1681 if detectRenames.lower() == "false" or detectRenames == "":
1682 self.diffOpts = ""
1683 elif detectRenames.lower() == "true":
1684 self.diffOpts = "-M"
1685 else:
1686 self.diffOpts = "-M%s" % detectRenames
1688 # no command-line arg for -C or --find-copies-harder, just
1689 # config variables
1690 detectCopies = gitConfig("git-p4.detectCopies")
1691 if detectCopies.lower() == "false" or detectCopies == "":
1692 pass
1693 elif detectCopies.lower() == "true":
1694 self.diffOpts += " -C"
1695 else:
1696 self.diffOpts += " -C%s" % detectCopies
1698 if gitConfigBool("git-p4.detectCopiesHarder"):
1699 self.diffOpts += " --find-copies-harder"
1702 # Apply the commits, one at a time. On failure, ask if should
1703 # continue to try the rest of the patches, or quit.
1705 if self.dry_run:
1706 print "Would apply"
1707 applied = []
1708 last = len(commits) - 1
1709 for i, commit in enumerate(commits):
1710 if self.dry_run:
1711 print " ", read_pipe(["git", "show", "-s",
1712 "--format=format:%h %s", commit])
1713 ok = True
1714 else:
1715 ok = self.applyCommit(commit)
1716 if ok:
1717 applied.append(commit)
1718 else:
1719 if self.prepare_p4_only and i < last:
1720 print "Processing only the first commit due to option" \
1721 " --prepare-p4-only"
1722 break
1723 if i < last:
1724 quit = False
1725 while True:
1726 # prompt for what to do, or use the option/variable
1727 if self.conflict_behavior == "ask":
1728 print "What do you want to do?"
1729 response = raw_input("[s]kip this commit but apply"
1730 " the rest, or [q]uit? ")
1731 if not response:
1732 continue
1733 elif self.conflict_behavior == "skip":
1734 response = "s"
1735 elif self.conflict_behavior == "quit":
1736 response = "q"
1737 else:
1738 die("Unknown conflict_behavior '%s'" %
1739 self.conflict_behavior)
1741 if response[0] == "s":
1742 print "Skipping this commit, but applying the rest"
1743 break
1744 if response[0] == "q":
1745 print "Quitting"
1746 quit = True
1747 break
1748 if quit:
1749 break
1751 chdir(self.oldWorkingDirectory)
1753 if self.dry_run:
1754 pass
1755 elif self.prepare_p4_only:
1756 pass
1757 elif len(commits) == len(applied):
1758 print "All commits applied!"
1760 sync = P4Sync()
1761 if self.branch:
1762 sync.branch = self.branch
1763 sync.run([])
1765 rebase = P4Rebase()
1766 rebase.rebase()
1768 else:
1769 if len(applied) == 0:
1770 print "No commits applied."
1771 else:
1772 print "Applied only the commits marked with '*':"
1773 for c in commits:
1774 if c in applied:
1775 star = "*"
1776 else:
1777 star = " "
1778 print star, read_pipe(["git", "show", "-s",
1779 "--format=format:%h %s", c])
1780 print "You will have to do 'git p4 sync' and rebase."
1782 if gitConfigBool("git-p4.exportLabels"):
1783 self.exportLabels = True
1785 if self.exportLabels:
1786 p4Labels = getP4Labels(self.depotPath)
1787 gitTags = getGitTags()
1789 missingGitTags = gitTags - p4Labels
1790 self.exportGitTags(missingGitTags)
1792 # exit with error unless everything applied perfectly
1793 if len(commits) != len(applied):
1794 sys.exit(1)
1796 return True
1798 class View(object):
1799 """Represent a p4 view ("p4 help views"), and map files in a
1800 repo according to the view."""
1802 def __init__(self, client_name):
1803 self.mappings = []
1804 self.client_prefix = "//%s/" % client_name
1805 # cache results of "p4 where" to lookup client file locations
1806 self.client_spec_path_cache = {}
1808 def append(self, view_line):
1809 """Parse a view line, splitting it into depot and client
1810 sides. Append to self.mappings, preserving order. This
1811 is only needed for tag creation."""
1813 # Split the view line into exactly two words. P4 enforces
1814 # structure on these lines that simplifies this quite a bit.
1816 # Either or both words may be double-quoted.
1817 # Single quotes do not matter.
1818 # Double-quote marks cannot occur inside the words.
1819 # A + or - prefix is also inside the quotes.
1820 # There are no quotes unless they contain a space.
1821 # The line is already white-space stripped.
1822 # The two words are separated by a single space.
1824 if view_line[0] == '"':
1825 # First word is double quoted. Find its end.
1826 close_quote_index = view_line.find('"', 1)
1827 if close_quote_index <= 0:
1828 die("No first-word closing quote found: %s" % view_line)
1829 depot_side = view_line[1:close_quote_index]
1830 # skip closing quote and space
1831 rhs_index = close_quote_index + 1 + 1
1832 else:
1833 space_index = view_line.find(" ")
1834 if space_index <= 0:
1835 die("No word-splitting space found: %s" % view_line)
1836 depot_side = view_line[0:space_index]
1837 rhs_index = space_index + 1
1839 # prefix + means overlay on previous mapping
1840 if depot_side.startswith("+"):
1841 depot_side = depot_side[1:]
1843 # prefix - means exclude this path, leave out of mappings
1844 exclude = False
1845 if depot_side.startswith("-"):
1846 exclude = True
1847 depot_side = depot_side[1:]
1849 if not exclude:
1850 self.mappings.append(depot_side)
1852 def convert_client_path(self, clientFile):
1853 # chop off //client/ part to make it relative
1854 if not clientFile.startswith(self.client_prefix):
1855 die("No prefix '%s' on clientFile '%s'" %
1856 (self.client_prefix, clientFile))
1857 return clientFile[len(self.client_prefix):]
1859 def update_client_spec_path_cache(self, files):
1860 """ Caching file paths by "p4 where" batch query """
1862 # List depot file paths exclude that already cached
1863 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1865 if len(fileArgs) == 0:
1866 return # All files in cache
1868 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1869 for res in where_result:
1870 if "code" in res and res["code"] == "error":
1871 # assume error is "... file(s) not in client view"
1872 continue
1873 if "clientFile" not in res:
1874 die("No clientFile from 'p4 where %s'" % depot_path)
1875 if "unmap" in res:
1876 # it will list all of them, but only one not unmap-ped
1877 continue
1878 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1880 # not found files or unmap files set to ""
1881 for depotFile in fileArgs:
1882 if depotFile not in self.client_spec_path_cache:
1883 self.client_spec_path_cache[depotFile] = ""
1885 def map_in_client(self, depot_path):
1886 """Return the relative location in the client where this
1887 depot file should live. Returns "" if the file should
1888 not be mapped in the client."""
1890 if depot_path in self.client_spec_path_cache:
1891 return self.client_spec_path_cache[depot_path]
1893 die( "Error: %s is not found in client spec path" % depot_path )
1894 return ""
1896 class P4Sync(Command, P4UserMap):
1897 delete_actions = ( "delete", "move/delete", "purge" )
1899 def __init__(self):
1900 Command.__init__(self)
1901 P4UserMap.__init__(self)
1902 self.options = [
1903 optparse.make_option("--branch", dest="branch"),
1904 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1905 optparse.make_option("--changesfile", dest="changesFile"),
1906 optparse.make_option("--silent", dest="silent", action="store_true"),
1907 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1908 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1909 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1910 help="Import into refs/heads/ , not refs/remotes"),
1911 optparse.make_option("--max-changes", dest="maxChanges"),
1912 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1913 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1914 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1915 help="Only sync files that are included in the Perforce Client Spec")
1917 self.description = """Imports from Perforce into a git repository.\n
1918 example:
1919 //depot/my/project/ -- to import the current head
1920 //depot/my/project/@all -- to import everything
1921 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1923 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1925 self.usage += " //depot/path[@revRange]"
1926 self.silent = False
1927 self.createdBranches = set()
1928 self.committedChanges = set()
1929 self.branch = ""
1930 self.detectBranches = False
1931 self.detectLabels = False
1932 self.importLabels = False
1933 self.changesFile = ""
1934 self.syncWithOrigin = True
1935 self.importIntoRemotes = True
1936 self.maxChanges = ""
1937 self.keepRepoPath = False
1938 self.depotPaths = None
1939 self.p4BranchesInGit = []
1940 self.cloneExclude = []
1941 self.useClientSpec = False
1942 self.useClientSpec_from_options = False
1943 self.clientSpecDirs = None
1944 self.tempBranches = []
1945 self.tempBranchLocation = "git-p4-tmp"
1947 if gitConfig("git-p4.syncFromOrigin") == "false":
1948 self.syncWithOrigin = False
1950 # Force a checkpoint in fast-import and wait for it to finish
1951 def checkpoint(self):
1952 self.gitStream.write("checkpoint\n\n")
1953 self.gitStream.write("progress checkpoint\n\n")
1954 out = self.gitOutput.readline()
1955 if self.verbose:
1956 print "checkpoint finished: " + out
1958 def extractFilesFromCommit(self, commit):
1959 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1960 for path in self.cloneExclude]
1961 files = []
1962 fnum = 0
1963 while commit.has_key("depotFile%s" % fnum):
1964 path = commit["depotFile%s" % fnum]
1966 if [p for p in self.cloneExclude
1967 if p4PathStartsWith(path, p)]:
1968 found = False
1969 else:
1970 found = [p for p in self.depotPaths
1971 if p4PathStartsWith(path, p)]
1972 if not found:
1973 fnum = fnum + 1
1974 continue
1976 file = {}
1977 file["path"] = path
1978 file["rev"] = commit["rev%s" % fnum]
1979 file["action"] = commit["action%s" % fnum]
1980 file["type"] = commit["type%s" % fnum]
1981 files.append(file)
1982 fnum = fnum + 1
1983 return files
1985 def stripRepoPath(self, path, prefixes):
1986 """When streaming files, this is called to map a p4 depot path
1987 to where it should go in git. The prefixes are either
1988 self.depotPaths, or self.branchPrefixes in the case of
1989 branch detection."""
1991 if self.useClientSpec:
1992 # branch detection moves files up a level (the branch name)
1993 # from what client spec interpretation gives
1994 path = self.clientSpecDirs.map_in_client(path)
1995 if self.detectBranches:
1996 for b in self.knownBranches:
1997 if path.startswith(b + "/"):
1998 path = path[len(b)+1:]
2000 elif self.keepRepoPath:
2001 # Preserve everything in relative path name except leading
2002 # //depot/; just look at first prefix as they all should
2003 # be in the same depot.
2004 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2005 if p4PathStartsWith(path, depot):
2006 path = path[len(depot):]
2008 else:
2009 for p in prefixes:
2010 if p4PathStartsWith(path, p):
2011 path = path[len(p):]
2012 break
2014 path = wildcard_decode(path)
2015 return path
2017 def splitFilesIntoBranches(self, commit):
2018 """Look at each depotFile in the commit to figure out to what
2019 branch it belongs."""
2021 if self.clientSpecDirs:
2022 files = self.extractFilesFromCommit(commit)
2023 self.clientSpecDirs.update_client_spec_path_cache(files)
2025 branches = {}
2026 fnum = 0
2027 while commit.has_key("depotFile%s" % fnum):
2028 path = commit["depotFile%s" % fnum]
2029 found = [p for p in self.depotPaths
2030 if p4PathStartsWith(path, p)]
2031 if not found:
2032 fnum = fnum + 1
2033 continue
2035 file = {}
2036 file["path"] = path
2037 file["rev"] = commit["rev%s" % fnum]
2038 file["action"] = commit["action%s" % fnum]
2039 file["type"] = commit["type%s" % fnum]
2040 fnum = fnum + 1
2042 # start with the full relative path where this file would
2043 # go in a p4 client
2044 if self.useClientSpec:
2045 relPath = self.clientSpecDirs.map_in_client(path)
2046 else:
2047 relPath = self.stripRepoPath(path, self.depotPaths)
2049 for branch in self.knownBranches.keys():
2050 # add a trailing slash so that a commit into qt/4.2foo
2051 # doesn't end up in qt/4.2, e.g.
2052 if relPath.startswith(branch + "/"):
2053 if branch not in branches:
2054 branches[branch] = []
2055 branches[branch].append(file)
2056 break
2058 return branches
2060 # output one file from the P4 stream
2061 # - helper for streamP4Files
2063 def streamOneP4File(self, file, contents):
2064 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2065 if verbose:
2066 sys.stderr.write("%s\n" % relPath)
2068 (type_base, type_mods) = split_p4_type(file["type"])
2070 git_mode = "100644"
2071 if "x" in type_mods:
2072 git_mode = "100755"
2073 if type_base == "symlink":
2074 git_mode = "120000"
2075 # p4 print on a symlink sometimes contains "target\n";
2076 # if it does, remove the newline
2077 data = ''.join(contents)
2078 if data[-1] == '\n':
2079 contents = [data[:-1]]
2080 else:
2081 contents = [data]
2083 if type_base == "utf16":
2084 # p4 delivers different text in the python output to -G
2085 # than it does when using "print -o", or normal p4 client
2086 # operations. utf16 is converted to ascii or utf8, perhaps.
2087 # But ascii text saved as -t utf16 is completely mangled.
2088 # Invoke print -o to get the real contents.
2090 # On windows, the newlines will always be mangled by print, so put
2091 # them back too. This is not needed to the cygwin windows version,
2092 # just the native "NT" type.
2094 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2095 if p4_version_string().find("/NT") >= 0:
2096 text = text.replace("\r\n", "\n")
2097 contents = [ text ]
2099 if type_base == "apple":
2100 # Apple filetype files will be streamed as a concatenation of
2101 # its appledouble header and the contents. This is useless
2102 # on both macs and non-macs. If using "print -q -o xx", it
2103 # will create "xx" with the data, and "%xx" with the header.
2104 # This is also not very useful.
2106 # Ideally, someday, this script can learn how to generate
2107 # appledouble files directly and import those to git, but
2108 # non-mac machines can never find a use for apple filetype.
2109 print "\nIgnoring apple filetype file %s" % file['depotFile']
2110 return
2112 # Note that we do not try to de-mangle keywords on utf16 files,
2113 # even though in theory somebody may want that.
2114 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2115 if pattern:
2116 regexp = re.compile(pattern, re.VERBOSE)
2117 text = ''.join(contents)
2118 text = regexp.sub(r'$\1$', text)
2119 contents = [ text ]
2121 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2123 # total length...
2124 length = 0
2125 for d in contents:
2126 length = length + len(d)
2128 self.gitStream.write("data %d\n" % length)
2129 for d in contents:
2130 self.gitStream.write(d)
2131 self.gitStream.write("\n")
2133 def streamOneP4Deletion(self, file):
2134 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2135 if verbose:
2136 sys.stderr.write("delete %s\n" % relPath)
2137 self.gitStream.write("D %s\n" % relPath)
2139 # handle another chunk of streaming data
2140 def streamP4FilesCb(self, marshalled):
2142 # catch p4 errors and complain
2143 err = None
2144 if "code" in marshalled:
2145 if marshalled["code"] == "error":
2146 if "data" in marshalled:
2147 err = marshalled["data"].rstrip()
2148 if err:
2149 f = None
2150 if self.stream_have_file_info:
2151 if "depotFile" in self.stream_file:
2152 f = self.stream_file["depotFile"]
2153 # force a failure in fast-import, else an empty
2154 # commit will be made
2155 self.gitStream.write("\n")
2156 self.gitStream.write("die-now\n")
2157 self.gitStream.close()
2158 # ignore errors, but make sure it exits first
2159 self.importProcess.wait()
2160 if f:
2161 die("Error from p4 print for %s: %s" % (f, err))
2162 else:
2163 die("Error from p4 print: %s" % err)
2165 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2166 # start of a new file - output the old one first
2167 self.streamOneP4File(self.stream_file, self.stream_contents)
2168 self.stream_file = {}
2169 self.stream_contents = []
2170 self.stream_have_file_info = False
2172 # pick up the new file information... for the
2173 # 'data' field we need to append to our array
2174 for k in marshalled.keys():
2175 if k == 'data':
2176 self.stream_contents.append(marshalled['data'])
2177 else:
2178 self.stream_file[k] = marshalled[k]
2180 self.stream_have_file_info = True
2182 # Stream directly from "p4 files" into "git fast-import"
2183 def streamP4Files(self, files):
2184 filesForCommit = []
2185 filesToRead = []
2186 filesToDelete = []
2188 for f in files:
2189 # if using a client spec, only add the files that have
2190 # a path in the client
2191 if self.clientSpecDirs:
2192 if self.clientSpecDirs.map_in_client(f['path']) == "":
2193 continue
2195 filesForCommit.append(f)
2196 if f['action'] in self.delete_actions:
2197 filesToDelete.append(f)
2198 else:
2199 filesToRead.append(f)
2201 # deleted files...
2202 for f in filesToDelete:
2203 self.streamOneP4Deletion(f)
2205 if len(filesToRead) > 0:
2206 self.stream_file = {}
2207 self.stream_contents = []
2208 self.stream_have_file_info = False
2210 # curry self argument
2211 def streamP4FilesCbSelf(entry):
2212 self.streamP4FilesCb(entry)
2214 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2216 p4CmdList(["-x", "-", "print"],
2217 stdin=fileArgs,
2218 cb=streamP4FilesCbSelf)
2220 # do the last chunk
2221 if self.stream_file.has_key('depotFile'):
2222 self.streamOneP4File(self.stream_file, self.stream_contents)
2224 def make_email(self, userid):
2225 if userid in self.users:
2226 return self.users[userid]
2227 else:
2228 return "%s <a@b>" % userid
2230 # Stream a p4 tag
2231 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2232 if verbose:
2233 print "writing tag %s for commit %s" % (labelName, commit)
2234 gitStream.write("tag %s\n" % labelName)
2235 gitStream.write("from %s\n" % commit)
2237 if labelDetails.has_key('Owner'):
2238 owner = labelDetails["Owner"]
2239 else:
2240 owner = None
2242 # Try to use the owner of the p4 label, or failing that,
2243 # the current p4 user id.
2244 if owner:
2245 email = self.make_email(owner)
2246 else:
2247 email = self.make_email(self.p4UserId())
2248 tagger = "%s %s %s" % (email, epoch, self.tz)
2250 gitStream.write("tagger %s\n" % tagger)
2252 print "labelDetails=",labelDetails
2253 if labelDetails.has_key('Description'):
2254 description = labelDetails['Description']
2255 else:
2256 description = 'Label from git p4'
2258 gitStream.write("data %d\n" % len(description))
2259 gitStream.write(description)
2260 gitStream.write("\n")
2262 def commit(self, details, files, branch, parent = ""):
2263 epoch = details["time"]
2264 author = details["user"]
2266 if self.verbose:
2267 print "commit into %s" % branch
2269 # start with reading files; if that fails, we should not
2270 # create a commit.
2271 new_files = []
2272 for f in files:
2273 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2274 new_files.append (f)
2275 else:
2276 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2278 if self.clientSpecDirs:
2279 self.clientSpecDirs.update_client_spec_path_cache(files)
2281 self.gitStream.write("commit %s\n" % branch)
2282 # gitStream.write("mark :%s\n" % details["change"])
2283 self.committedChanges.add(int(details["change"]))
2284 committer = ""
2285 if author not in self.users:
2286 self.getUserMapFromPerforceServer()
2287 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2289 self.gitStream.write("committer %s\n" % committer)
2291 self.gitStream.write("data <<EOT\n")
2292 self.gitStream.write(details["desc"])
2293 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2294 (','.join(self.branchPrefixes), details["change"]))
2295 if len(details['options']) > 0:
2296 self.gitStream.write(": options = %s" % details['options'])
2297 self.gitStream.write("]\nEOT\n\n")
2299 if len(parent) > 0:
2300 if self.verbose:
2301 print "parent %s" % parent
2302 self.gitStream.write("from %s\n" % parent)
2304 self.streamP4Files(new_files)
2305 self.gitStream.write("\n")
2307 change = int(details["change"])
2309 if self.labels.has_key(change):
2310 label = self.labels[change]
2311 labelDetails = label[0]
2312 labelRevisions = label[1]
2313 if self.verbose:
2314 print "Change %s is labelled %s" % (change, labelDetails)
2316 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2317 for p in self.branchPrefixes])
2319 if len(files) == len(labelRevisions):
2321 cleanedFiles = {}
2322 for info in files:
2323 if info["action"] in self.delete_actions:
2324 continue
2325 cleanedFiles[info["depotFile"]] = info["rev"]
2327 if cleanedFiles == labelRevisions:
2328 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2330 else:
2331 if not self.silent:
2332 print ("Tag %s does not match with change %s: files do not match."
2333 % (labelDetails["label"], change))
2335 else:
2336 if not self.silent:
2337 print ("Tag %s does not match with change %s: file count is different."
2338 % (labelDetails["label"], change))
2340 # Build a dictionary of changelists and labels, for "detect-labels" option.
2341 def getLabels(self):
2342 self.labels = {}
2344 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2345 if len(l) > 0 and not self.silent:
2346 print "Finding files belonging to labels in %s" % `self.depotPaths`
2348 for output in l:
2349 label = output["label"]
2350 revisions = {}
2351 newestChange = 0
2352 if self.verbose:
2353 print "Querying files for label %s" % label
2354 for file in p4CmdList(["files"] +
2355 ["%s...@%s" % (p, label)
2356 for p in self.depotPaths]):
2357 revisions[file["depotFile"]] = file["rev"]
2358 change = int(file["change"])
2359 if change > newestChange:
2360 newestChange = change
2362 self.labels[newestChange] = [output, revisions]
2364 if self.verbose:
2365 print "Label changes: %s" % self.labels.keys()
2367 # Import p4 labels as git tags. A direct mapping does not
2368 # exist, so assume that if all the files are at the same revision
2369 # then we can use that, or it's something more complicated we should
2370 # just ignore.
2371 def importP4Labels(self, stream, p4Labels):
2372 if verbose:
2373 print "import p4 labels: " + ' '.join(p4Labels)
2375 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2376 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2377 if len(validLabelRegexp) == 0:
2378 validLabelRegexp = defaultLabelRegexp
2379 m = re.compile(validLabelRegexp)
2381 for name in p4Labels:
2382 commitFound = False
2384 if not m.match(name):
2385 if verbose:
2386 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2387 continue
2389 if name in ignoredP4Labels:
2390 continue
2392 labelDetails = p4CmdList(['label', "-o", name])[0]
2394 # get the most recent changelist for each file in this label
2395 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2396 for p in self.depotPaths])
2398 if change.has_key('change'):
2399 # find the corresponding git commit; take the oldest commit
2400 changelist = int(change['change'])
2401 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2402 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2403 if len(gitCommit) == 0:
2404 print "could not find git commit for changelist %d" % changelist
2405 else:
2406 gitCommit = gitCommit.strip()
2407 commitFound = True
2408 # Convert from p4 time format
2409 try:
2410 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2411 except ValueError:
2412 print "Could not convert label time %s" % labelDetails['Update']
2413 tmwhen = 1
2415 when = int(time.mktime(tmwhen))
2416 self.streamTag(stream, name, labelDetails, gitCommit, when)
2417 if verbose:
2418 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2419 else:
2420 if verbose:
2421 print "Label %s has no changelists - possibly deleted?" % name
2423 if not commitFound:
2424 # We can't import this label; don't try again as it will get very
2425 # expensive repeatedly fetching all the files for labels that will
2426 # never be imported. If the label is moved in the future, the
2427 # ignore will need to be removed manually.
2428 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2430 def guessProjectName(self):
2431 for p in self.depotPaths:
2432 if p.endswith("/"):
2433 p = p[:-1]
2434 p = p[p.strip().rfind("/") + 1:]
2435 if not p.endswith("/"):
2436 p += "/"
2437 return p
2439 def getBranchMapping(self):
2440 lostAndFoundBranches = set()
2442 user = gitConfig("git-p4.branchUser")
2443 if len(user) > 0:
2444 command = "branches -u %s" % user
2445 else:
2446 command = "branches"
2448 for info in p4CmdList(command):
2449 details = p4Cmd(["branch", "-o", info["branch"]])
2450 viewIdx = 0
2451 while details.has_key("View%s" % viewIdx):
2452 paths = details["View%s" % viewIdx].split(" ")
2453 viewIdx = viewIdx + 1
2454 # require standard //depot/foo/... //depot/bar/... mapping
2455 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2456 continue
2457 source = paths[0]
2458 destination = paths[1]
2459 ## HACK
2460 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2461 source = source[len(self.depotPaths[0]):-4]
2462 destination = destination[len(self.depotPaths[0]):-4]
2464 if destination in self.knownBranches:
2465 if not self.silent:
2466 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2467 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2468 continue
2470 self.knownBranches[destination] = source
2472 lostAndFoundBranches.discard(destination)
2474 if source not in self.knownBranches:
2475 lostAndFoundBranches.add(source)
2477 # Perforce does not strictly require branches to be defined, so we also
2478 # check git config for a branch list.
2480 # Example of branch definition in git config file:
2481 # [git-p4]
2482 # branchList=main:branchA
2483 # branchList=main:branchB
2484 # branchList=branchA:branchC
2485 configBranches = gitConfigList("git-p4.branchList")
2486 for branch in configBranches:
2487 if branch:
2488 (source, destination) = branch.split(":")
2489 self.knownBranches[destination] = source
2491 lostAndFoundBranches.discard(destination)
2493 if source not in self.knownBranches:
2494 lostAndFoundBranches.add(source)
2497 for branch in lostAndFoundBranches:
2498 self.knownBranches[branch] = branch
2500 def getBranchMappingFromGitBranches(self):
2501 branches = p4BranchesInGit(self.importIntoRemotes)
2502 for branch in branches.keys():
2503 if branch == "master":
2504 branch = "main"
2505 else:
2506 branch = branch[len(self.projectName):]
2507 self.knownBranches[branch] = branch
2509 def updateOptionDict(self, d):
2510 option_keys = {}
2511 if self.keepRepoPath:
2512 option_keys['keepRepoPath'] = 1
2514 d["options"] = ' '.join(sorted(option_keys.keys()))
2516 def readOptions(self, d):
2517 self.keepRepoPath = (d.has_key('options')
2518 and ('keepRepoPath' in d['options']))
2520 def gitRefForBranch(self, branch):
2521 if branch == "main":
2522 return self.refPrefix + "master"
2524 if len(branch) <= 0:
2525 return branch
2527 return self.refPrefix + self.projectName + branch
2529 def gitCommitByP4Change(self, ref, change):
2530 if self.verbose:
2531 print "looking in ref " + ref + " for change %s using bisect..." % change
2533 earliestCommit = ""
2534 latestCommit = parseRevision(ref)
2536 while True:
2537 if self.verbose:
2538 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2539 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2540 if len(next) == 0:
2541 if self.verbose:
2542 print "argh"
2543 return ""
2544 log = extractLogMessageFromGitCommit(next)
2545 settings = extractSettingsGitLog(log)
2546 currentChange = int(settings['change'])
2547 if self.verbose:
2548 print "current change %s" % currentChange
2550 if currentChange == change:
2551 if self.verbose:
2552 print "found %s" % next
2553 return next
2555 if currentChange < change:
2556 earliestCommit = "^%s" % next
2557 else:
2558 latestCommit = "%s" % next
2560 return ""
2562 def importNewBranch(self, branch, maxChange):
2563 # make fast-import flush all changes to disk and update the refs using the checkpoint
2564 # command so that we can try to find the branch parent in the git history
2565 self.gitStream.write("checkpoint\n\n");
2566 self.gitStream.flush();
2567 branchPrefix = self.depotPaths[0] + branch + "/"
2568 range = "@1,%s" % maxChange
2569 #print "prefix" + branchPrefix
2570 changes = p4ChangesForPaths([branchPrefix], range)
2571 if len(changes) <= 0:
2572 return False
2573 firstChange = changes[0]
2574 #print "first change in branch: %s" % firstChange
2575 sourceBranch = self.knownBranches[branch]
2576 sourceDepotPath = self.depotPaths[0] + sourceBranch
2577 sourceRef = self.gitRefForBranch(sourceBranch)
2578 #print "source " + sourceBranch
2580 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2581 #print "branch parent: %s" % branchParentChange
2582 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2583 if len(gitParent) > 0:
2584 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2585 #print "parent git commit: %s" % gitParent
2587 self.importChanges(changes)
2588 return True
2590 def searchParent(self, parent, branch, target):
2591 parentFound = False
2592 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2593 "--no-merges", parent]):
2594 blob = blob.strip()
2595 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2596 parentFound = True
2597 if self.verbose:
2598 print "Found parent of %s in commit %s" % (branch, blob)
2599 break
2600 if parentFound:
2601 return blob
2602 else:
2603 return None
2605 def importChanges(self, changes):
2606 cnt = 1
2607 for change in changes:
2608 description = p4_describe(change)
2609 self.updateOptionDict(description)
2611 if not self.silent:
2612 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2613 sys.stdout.flush()
2614 cnt = cnt + 1
2616 try:
2617 if self.detectBranches:
2618 branches = self.splitFilesIntoBranches(description)
2619 for branch in branches.keys():
2620 ## HACK --hwn
2621 branchPrefix = self.depotPaths[0] + branch + "/"
2622 self.branchPrefixes = [ branchPrefix ]
2624 parent = ""
2626 filesForCommit = branches[branch]
2628 if self.verbose:
2629 print "branch is %s" % branch
2631 self.updatedBranches.add(branch)
2633 if branch not in self.createdBranches:
2634 self.createdBranches.add(branch)
2635 parent = self.knownBranches[branch]
2636 if parent == branch:
2637 parent = ""
2638 else:
2639 fullBranch = self.projectName + branch
2640 if fullBranch not in self.p4BranchesInGit:
2641 if not self.silent:
2642 print("\n Importing new branch %s" % fullBranch);
2643 if self.importNewBranch(branch, change - 1):
2644 parent = ""
2645 self.p4BranchesInGit.append(fullBranch)
2646 if not self.silent:
2647 print("\n Resuming with change %s" % change);
2649 if self.verbose:
2650 print "parent determined through known branches: %s" % parent
2652 branch = self.gitRefForBranch(branch)
2653 parent = self.gitRefForBranch(parent)
2655 if self.verbose:
2656 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2658 if len(parent) == 0 and branch in self.initialParents:
2659 parent = self.initialParents[branch]
2660 del self.initialParents[branch]
2662 blob = None
2663 if len(parent) > 0:
2664 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2665 if self.verbose:
2666 print "Creating temporary branch: " + tempBranch
2667 self.commit(description, filesForCommit, tempBranch)
2668 self.tempBranches.append(tempBranch)
2669 self.checkpoint()
2670 blob = self.searchParent(parent, branch, tempBranch)
2671 if blob:
2672 self.commit(description, filesForCommit, branch, blob)
2673 else:
2674 if self.verbose:
2675 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2676 self.commit(description, filesForCommit, branch, parent)
2677 else:
2678 files = self.extractFilesFromCommit(description)
2679 self.commit(description, files, self.branch,
2680 self.initialParent)
2681 # only needed once, to connect to the previous commit
2682 self.initialParent = ""
2683 except IOError:
2684 print self.gitError.read()
2685 sys.exit(1)
2687 def importHeadRevision(self, revision):
2688 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2690 details = {}
2691 details["user"] = "git perforce import user"
2692 details["desc"] = ("Initial import of %s from the state at revision %s\n"
2693 % (' '.join(self.depotPaths), revision))
2694 details["change"] = revision
2695 newestRevision = 0
2697 fileCnt = 0
2698 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2700 for info in p4CmdList(["files"] + fileArgs):
2702 if 'code' in info and info['code'] == 'error':
2703 sys.stderr.write("p4 returned an error: %s\n"
2704 % info['data'])
2705 if info['data'].find("must refer to client") >= 0:
2706 sys.stderr.write("This particular p4 error is misleading.\n")
2707 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2708 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
2709 sys.exit(1)
2710 if 'p4ExitCode' in info:
2711 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2712 sys.exit(1)
2715 change = int(info["change"])
2716 if change > newestRevision:
2717 newestRevision = change
2719 if info["action"] in self.delete_actions:
2720 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2721 #fileCnt = fileCnt + 1
2722 continue
2724 for prop in ["depotFile", "rev", "action", "type" ]:
2725 details["%s%s" % (prop, fileCnt)] = info[prop]
2727 fileCnt = fileCnt + 1
2729 details["change"] = newestRevision
2731 # Use time from top-most change so that all git p4 clones of
2732 # the same p4 repo have the same commit SHA1s.
2733 res = p4_describe(newestRevision)
2734 details["time"] = res["time"]
2736 self.updateOptionDict(details)
2737 try:
2738 self.commit(details, self.extractFilesFromCommit(details), self.branch)
2739 except IOError:
2740 print "IO error with git fast-import. Is your git version recent enough?"
2741 print self.gitError.read()
2744 def run(self, args):
2745 self.depotPaths = []
2746 self.changeRange = ""
2747 self.previousDepotPaths = []
2748 self.hasOrigin = False
2750 # map from branch depot path to parent branch
2751 self.knownBranches = {}
2752 self.initialParents = {}
2754 if self.importIntoRemotes:
2755 self.refPrefix = "refs/remotes/p4/"
2756 else:
2757 self.refPrefix = "refs/heads/p4/"
2759 if self.syncWithOrigin:
2760 self.hasOrigin = originP4BranchesExist()
2761 if self.hasOrigin:
2762 if not self.silent:
2763 print 'Syncing with origin first, using "git fetch origin"'
2764 system("git fetch origin")
2766 branch_arg_given = bool(self.branch)
2767 if len(self.branch) == 0:
2768 self.branch = self.refPrefix + "master"
2769 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2770 system("git update-ref %s refs/heads/p4" % self.branch)
2771 system("git branch -D p4")
2773 # accept either the command-line option, or the configuration variable
2774 if self.useClientSpec:
2775 # will use this after clone to set the variable
2776 self.useClientSpec_from_options = True
2777 else:
2778 if gitConfigBool("git-p4.useclientspec"):
2779 self.useClientSpec = True
2780 if self.useClientSpec:
2781 self.clientSpecDirs = getClientSpec()
2783 # TODO: should always look at previous commits,
2784 # merge with previous imports, if possible.
2785 if args == []:
2786 if self.hasOrigin:
2787 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2789 # branches holds mapping from branch name to sha1
2790 branches = p4BranchesInGit(self.importIntoRemotes)
2792 # restrict to just this one, disabling detect-branches
2793 if branch_arg_given:
2794 short = self.branch.split("/")[-1]
2795 if short in branches:
2796 self.p4BranchesInGit = [ short ]
2797 else:
2798 self.p4BranchesInGit = branches.keys()
2800 if len(self.p4BranchesInGit) > 1:
2801 if not self.silent:
2802 print "Importing from/into multiple branches"
2803 self.detectBranches = True
2804 for branch in branches.keys():
2805 self.initialParents[self.refPrefix + branch] = \
2806 branches[branch]
2808 if self.verbose:
2809 print "branches: %s" % self.p4BranchesInGit
2811 p4Change = 0
2812 for branch in self.p4BranchesInGit:
2813 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
2815 settings = extractSettingsGitLog(logMsg)
2817 self.readOptions(settings)
2818 if (settings.has_key('depot-paths')
2819 and settings.has_key ('change')):
2820 change = int(settings['change']) + 1
2821 p4Change = max(p4Change, change)
2823 depotPaths = sorted(settings['depot-paths'])
2824 if self.previousDepotPaths == []:
2825 self.previousDepotPaths = depotPaths
2826 else:
2827 paths = []
2828 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2829 prev_list = prev.split("/")
2830 cur_list = cur.split("/")
2831 for i in range(0, min(len(cur_list), len(prev_list))):
2832 if cur_list[i] <> prev_list[i]:
2833 i = i - 1
2834 break
2836 paths.append ("/".join(cur_list[:i + 1]))
2838 self.previousDepotPaths = paths
2840 if p4Change > 0:
2841 self.depotPaths = sorted(self.previousDepotPaths)
2842 self.changeRange = "@%s,#head" % p4Change
2843 if not self.silent and not self.detectBranches:
2844 print "Performing incremental import into %s git branch" % self.branch
2846 # accept multiple ref name abbreviations:
2847 # refs/foo/bar/branch -> use it exactly
2848 # p4/branch -> prepend refs/remotes/ or refs/heads/
2849 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2850 if not self.branch.startswith("refs/"):
2851 if self.importIntoRemotes:
2852 prepend = "refs/remotes/"
2853 else:
2854 prepend = "refs/heads/"
2855 if not self.branch.startswith("p4/"):
2856 prepend += "p4/"
2857 self.branch = prepend + self.branch
2859 if len(args) == 0 and self.depotPaths:
2860 if not self.silent:
2861 print "Depot paths: %s" % ' '.join(self.depotPaths)
2862 else:
2863 if self.depotPaths and self.depotPaths != args:
2864 print ("previous import used depot path %s and now %s was specified. "
2865 "This doesn't work!" % (' '.join (self.depotPaths),
2866 ' '.join (args)))
2867 sys.exit(1)
2869 self.depotPaths = sorted(args)
2871 revision = ""
2872 self.users = {}
2874 # Make sure no revision specifiers are used when --changesfile
2875 # is specified.
2876 bad_changesfile = False
2877 if len(self.changesFile) > 0:
2878 for p in self.depotPaths:
2879 if p.find("@") >= 0 or p.find("#") >= 0:
2880 bad_changesfile = True
2881 break
2882 if bad_changesfile:
2883 die("Option --changesfile is incompatible with revision specifiers")
2885 newPaths = []
2886 for p in self.depotPaths:
2887 if p.find("@") != -1:
2888 atIdx = p.index("@")
2889 self.changeRange = p[atIdx:]
2890 if self.changeRange == "@all":
2891 self.changeRange = ""
2892 elif ',' not in self.changeRange:
2893 revision = self.changeRange
2894 self.changeRange = ""
2895 p = p[:atIdx]
2896 elif p.find("#") != -1:
2897 hashIdx = p.index("#")
2898 revision = p[hashIdx:]
2899 p = p[:hashIdx]
2900 elif self.previousDepotPaths == []:
2901 # pay attention to changesfile, if given, else import
2902 # the entire p4 tree at the head revision
2903 if len(self.changesFile) == 0:
2904 revision = "#head"
2906 p = re.sub ("\.\.\.$", "", p)
2907 if not p.endswith("/"):
2908 p += "/"
2910 newPaths.append(p)
2912 self.depotPaths = newPaths
2914 # --detect-branches may change this for each branch
2915 self.branchPrefixes = self.depotPaths
2917 self.loadUserMapFromCache()
2918 self.labels = {}
2919 if self.detectLabels:
2920 self.getLabels();
2922 if self.detectBranches:
2923 ## FIXME - what's a P4 projectName ?
2924 self.projectName = self.guessProjectName()
2926 if self.hasOrigin:
2927 self.getBranchMappingFromGitBranches()
2928 else:
2929 self.getBranchMapping()
2930 if self.verbose:
2931 print "p4-git branches: %s" % self.p4BranchesInGit
2932 print "initial parents: %s" % self.initialParents
2933 for b in self.p4BranchesInGit:
2934 if b != "master":
2936 ## FIXME
2937 b = b[len(self.projectName):]
2938 self.createdBranches.add(b)
2940 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2942 self.importProcess = subprocess.Popen(["git", "fast-import"],
2943 stdin=subprocess.PIPE,
2944 stdout=subprocess.PIPE,
2945 stderr=subprocess.PIPE);
2946 self.gitOutput = self.importProcess.stdout
2947 self.gitStream = self.importProcess.stdin
2948 self.gitError = self.importProcess.stderr
2950 if revision:
2951 self.importHeadRevision(revision)
2952 else:
2953 changes = []
2955 if len(self.changesFile) > 0:
2956 output = open(self.changesFile).readlines()
2957 changeSet = set()
2958 for line in output:
2959 changeSet.add(int(line))
2961 for change in changeSet:
2962 changes.append(change)
2964 changes.sort()
2965 else:
2966 # catch "git p4 sync" with no new branches, in a repo that
2967 # does not have any existing p4 branches
2968 if len(args) == 0:
2969 if not self.p4BranchesInGit:
2970 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
2972 # The default branch is master, unless --branch is used to
2973 # specify something else. Make sure it exists, or complain
2974 # nicely about how to use --branch.
2975 if not self.detectBranches:
2976 if not branch_exists(self.branch):
2977 if branch_arg_given:
2978 die("Error: branch %s does not exist." % self.branch)
2979 else:
2980 die("Error: no branch %s; perhaps specify one with --branch." %
2981 self.branch)
2983 if self.verbose:
2984 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2985 self.changeRange)
2986 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2988 if len(self.maxChanges) > 0:
2989 changes = changes[:min(int(self.maxChanges), len(changes))]
2991 if len(changes) == 0:
2992 if not self.silent:
2993 print "No changes to import!"
2994 else:
2995 if not self.silent and not self.detectBranches:
2996 print "Import destination: %s" % self.branch
2998 self.updatedBranches = set()
3000 if not self.detectBranches:
3001 if args:
3002 # start a new branch
3003 self.initialParent = ""
3004 else:
3005 # build on a previous revision
3006 self.initialParent = parseRevision(self.branch)
3008 self.importChanges(changes)
3010 if not self.silent:
3011 print ""
3012 if len(self.updatedBranches) > 0:
3013 sys.stdout.write("Updated branches: ")
3014 for b in self.updatedBranches:
3015 sys.stdout.write("%s " % b)
3016 sys.stdout.write("\n")
3018 if gitConfigBool("git-p4.importLabels"):
3019 self.importLabels = True
3021 if self.importLabels:
3022 p4Labels = getP4Labels(self.depotPaths)
3023 gitTags = getGitTags()
3025 missingP4Labels = p4Labels - gitTags
3026 self.importP4Labels(self.gitStream, missingP4Labels)
3028 self.gitStream.close()
3029 if self.importProcess.wait() != 0:
3030 die("fast-import failed: %s" % self.gitError.read())
3031 self.gitOutput.close()
3032 self.gitError.close()
3034 # Cleanup temporary branches created during import
3035 if self.tempBranches != []:
3036 for branch in self.tempBranches:
3037 read_pipe("git update-ref -d %s" % branch)
3038 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3040 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3041 # a convenient shortcut refname "p4".
3042 if self.importIntoRemotes:
3043 head_ref = self.refPrefix + "HEAD"
3044 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3045 system(["git", "symbolic-ref", head_ref, self.branch])
3047 return True
3049 class P4Rebase(Command):
3050 def __init__(self):
3051 Command.__init__(self)
3052 self.options = [
3053 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3055 self.importLabels = False
3056 self.description = ("Fetches the latest revision from perforce and "
3057 + "rebases the current work (branch) against it")
3059 def run(self, args):
3060 sync = P4Sync()
3061 sync.importLabels = self.importLabels
3062 sync.run([])
3064 return self.rebase()
3066 def rebase(self):
3067 if os.system("git update-index --refresh") != 0:
3068 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3069 if len(read_pipe("git diff-index HEAD --")) > 0:
3070 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3072 [upstream, settings] = findUpstreamBranchPoint()
3073 if len(upstream) == 0:
3074 die("Cannot find upstream branchpoint for rebase")
3076 # the branchpoint may be p4/foo~3, so strip off the parent
3077 upstream = re.sub("~[0-9]+$", "", upstream)
3079 print "Rebasing the current branch onto %s" % upstream
3080 oldHead = read_pipe("git rev-parse HEAD").strip()
3081 system("git rebase %s" % upstream)
3082 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3083 return True
3085 class P4Clone(P4Sync):
3086 def __init__(self):
3087 P4Sync.__init__(self)
3088 self.description = "Creates a new git repository and imports from Perforce into it"
3089 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3090 self.options += [
3091 optparse.make_option("--destination", dest="cloneDestination",
3092 action='store', default=None,
3093 help="where to leave result of the clone"),
3094 optparse.make_option("-/", dest="cloneExclude",
3095 action="append", type="string",
3096 help="exclude depot path"),
3097 optparse.make_option("--bare", dest="cloneBare",
3098 action="store_true", default=False),
3100 self.cloneDestination = None
3101 self.needsGit = False
3102 self.cloneBare = False
3104 # This is required for the "append" cloneExclude action
3105 def ensure_value(self, attr, value):
3106 if not hasattr(self, attr) or getattr(self, attr) is None:
3107 setattr(self, attr, value)
3108 return getattr(self, attr)
3110 def defaultDestination(self, args):
3111 ## TODO: use common prefix of args?
3112 depotPath = args[0]
3113 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3114 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3115 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3116 depotDir = re.sub(r"/$", "", depotDir)
3117 return os.path.split(depotDir)[1]
3119 def run(self, args):
3120 if len(args) < 1:
3121 return False
3123 if self.keepRepoPath and not self.cloneDestination:
3124 sys.stderr.write("Must specify destination for --keep-path\n")
3125 sys.exit(1)
3127 depotPaths = args
3129 if not self.cloneDestination and len(depotPaths) > 1:
3130 self.cloneDestination = depotPaths[-1]
3131 depotPaths = depotPaths[:-1]
3133 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3134 for p in depotPaths:
3135 if not p.startswith("//"):
3136 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3137 return False
3139 if not self.cloneDestination:
3140 self.cloneDestination = self.defaultDestination(args)
3142 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3144 if not os.path.exists(self.cloneDestination):
3145 os.makedirs(self.cloneDestination)
3146 chdir(self.cloneDestination)
3148 init_cmd = [ "git", "init" ]
3149 if self.cloneBare:
3150 init_cmd.append("--bare")
3151 retcode = subprocess.call(init_cmd)
3152 if retcode:
3153 raise CalledProcessError(retcode, init_cmd)
3155 if not P4Sync.run(self, depotPaths):
3156 return False
3158 # create a master branch and check out a work tree
3159 if gitBranchExists(self.branch):
3160 system([ "git", "branch", "master", self.branch ])
3161 if not self.cloneBare:
3162 system([ "git", "checkout", "-f" ])
3163 else:
3164 print 'Not checking out any branch, use ' \
3165 '"git checkout -q -b master <branch>"'
3167 # auto-set this variable if invoked with --use-client-spec
3168 if self.useClientSpec_from_options:
3169 system("git config --bool git-p4.useclientspec true")
3171 return True
3173 class P4Branches(Command):
3174 def __init__(self):
3175 Command.__init__(self)
3176 self.options = [ ]
3177 self.description = ("Shows the git branches that hold imports and their "
3178 + "corresponding perforce depot paths")
3179 self.verbose = False
3181 def run(self, args):
3182 if originP4BranchesExist():
3183 createOrUpdateBranchesFromOrigin()
3185 cmdline = "git rev-parse --symbolic "
3186 cmdline += " --remotes"
3188 for line in read_pipe_lines(cmdline):
3189 line = line.strip()
3191 if not line.startswith('p4/') or line == "p4/HEAD":
3192 continue
3193 branch = line
3195 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3196 settings = extractSettingsGitLog(log)
3198 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3199 return True
3201 class HelpFormatter(optparse.IndentedHelpFormatter):
3202 def __init__(self):
3203 optparse.IndentedHelpFormatter.__init__(self)
3205 def format_description(self, description):
3206 if description:
3207 return description + "\n"
3208 else:
3209 return ""
3211 def printUsage(commands):
3212 print "usage: %s <command> [options]" % sys.argv[0]
3213 print ""
3214 print "valid commands: %s" % ", ".join(commands)
3215 print ""
3216 print "Try %s <command> --help for command specific help." % sys.argv[0]
3217 print ""
3219 commands = {
3220 "debug" : P4Debug,
3221 "submit" : P4Submit,
3222 "commit" : P4Submit,
3223 "sync" : P4Sync,
3224 "rebase" : P4Rebase,
3225 "clone" : P4Clone,
3226 "rollback" : P4RollBack,
3227 "branches" : P4Branches
3231 def main():
3232 if len(sys.argv[1:]) == 0:
3233 printUsage(commands.keys())
3234 sys.exit(2)
3236 cmdName = sys.argv[1]
3237 try:
3238 klass = commands[cmdName]
3239 cmd = klass()
3240 except KeyError:
3241 print "unknown command %s" % cmdName
3242 print ""
3243 printUsage(commands.keys())
3244 sys.exit(2)
3246 options = cmd.options
3247 cmd.gitdir = os.environ.get("GIT_DIR", None)
3249 args = sys.argv[2:]
3251 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3252 if cmd.needsGit:
3253 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3255 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3256 options,
3257 description = cmd.description,
3258 formatter = HelpFormatter())
3260 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3261 global verbose
3262 verbose = cmd.verbose
3263 if cmd.needsGit:
3264 if cmd.gitdir == None:
3265 cmd.gitdir = os.path.abspath(".git")
3266 if not isValidGitDir(cmd.gitdir):
3267 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3268 if os.path.exists(cmd.gitdir):
3269 cdup = read_pipe("git rev-parse --show-cdup").strip()
3270 if len(cdup) > 0:
3271 chdir(cdup);
3273 if not isValidGitDir(cmd.gitdir):
3274 if isValidGitDir(cmd.gitdir + "/.git"):
3275 cmd.gitdir += "/.git"
3276 else:
3277 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3279 os.environ["GIT_DIR"] = cmd.gitdir
3281 if not cmd.run(args):
3282 parser.print_help()
3283 sys.exit(2)
3286 if __name__ == '__main__':
3287 main()