Merge branch 'jc/lock-report-on-error'
[git.git] / git-p4.py
blob22e3f57e7d93783ef6bb3f0bfd4ef2f6bbd0f93f
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
30 try:
31 from subprocess import CalledProcessError
32 except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
45 verbose = False
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
53 def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
60 real_cmd = ["p4"]
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
64 real_cmd += ["-u",user]
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
68 real_cmd += ["-P", password]
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
72 real_cmd += ["-p", port]
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
76 real_cmd += ["-H", host]
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
80 real_cmd += ["-c", client]
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
86 real_cmd += ["-r", str(retries)]
88 if isinstance(cmd,basestring):
89 real_cmd = ' '.join(real_cmd) + ' ' + cmd
90 else:
91 real_cmd += cmd
92 return real_cmd
94 def git_dir(path):
95 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
96 This won't automatically add ".git" to a directory.
97 """
98 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
99 if not d or len(d) == 0:
100 return None
101 else:
102 return d
104 def chdir(path, is_client_path=False):
105 """Do chdir to the given path, and set the PWD environment
106 variable for use by P4. It does not look at getcwd() output.
107 Since we're not using the shell, it is necessary to set the
108 PWD environment variable explicitly.
110 Normally, expand the path to force it to be absolute. This
111 addresses the use of relative path names inside P4 settings,
112 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
113 as given; it looks for .p4config using PWD.
115 If is_client_path, the path was handed to us directly by p4,
116 and may be a symbolic link. Do not call os.getcwd() in this
117 case, because it will cause p4 to think that PWD is not inside
118 the client path.
121 os.chdir(path)
122 if not is_client_path:
123 path = os.getcwd()
124 os.environ['PWD'] = path
126 def calcDiskFree():
127 """Return free space in bytes on the disk of the given dirname."""
128 if platform.system() == 'Windows':
129 free_bytes = ctypes.c_ulonglong(0)
130 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
131 return free_bytes.value
132 else:
133 st = os.statvfs(os.getcwd())
134 return st.f_bavail * st.f_frsize
136 def die(msg):
137 if verbose:
138 raise Exception(msg)
139 else:
140 sys.stderr.write(msg + "\n")
141 sys.exit(1)
143 def write_pipe(c, stdin):
144 if verbose:
145 sys.stderr.write('Writing pipe: %s\n' % str(c))
147 expand = isinstance(c,basestring)
148 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
149 pipe = p.stdin
150 val = pipe.write(stdin)
151 pipe.close()
152 if p.wait():
153 die('Command failed: %s' % str(c))
155 return val
157 def p4_write_pipe(c, stdin):
158 real_cmd = p4_build_cmd(c)
159 return write_pipe(real_cmd, stdin)
161 def read_pipe(c, ignore_error=False):
162 if verbose:
163 sys.stderr.write('Reading pipe: %s\n' % str(c))
165 expand = isinstance(c,basestring)
166 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
167 (out, err) = p.communicate()
168 if p.returncode != 0 and not ignore_error:
169 die('Command failed: %s\nError: %s' % (str(c), err))
170 return out
172 def p4_read_pipe(c, ignore_error=False):
173 real_cmd = p4_build_cmd(c)
174 return read_pipe(real_cmd, ignore_error)
176 def read_pipe_lines(c):
177 if verbose:
178 sys.stderr.write('Reading pipe: %s\n' % str(c))
180 expand = isinstance(c, basestring)
181 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
182 pipe = p.stdout
183 val = pipe.readlines()
184 if pipe.close() or p.wait():
185 die('Command failed: %s' % str(c))
187 return val
189 def p4_read_pipe_lines(c):
190 """Specifically invoke p4 on the command supplied. """
191 real_cmd = p4_build_cmd(c)
192 return read_pipe_lines(real_cmd)
194 def p4_has_command(cmd):
195 """Ask p4 for help on this command. If it returns an error, the
196 command does not exist in this version of p4."""
197 real_cmd = p4_build_cmd(["help", cmd])
198 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
199 stderr=subprocess.PIPE)
200 p.communicate()
201 return p.returncode == 0
203 def p4_has_move_command():
204 """See if the move command exists, that it supports -k, and that
205 it has not been administratively disabled. The arguments
206 must be correct, but the filenames do not have to exist. Use
207 ones with wildcards so even if they exist, it will fail."""
209 if not p4_has_command("move"):
210 return False
211 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
212 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
213 (out, err) = p.communicate()
214 # return code will be 1 in either case
215 if err.find("Invalid option") >= 0:
216 return False
217 if err.find("disabled") >= 0:
218 return False
219 # assume it failed because @... was invalid changelist
220 return True
222 def system(cmd, ignore_error=False):
223 expand = isinstance(cmd,basestring)
224 if verbose:
225 sys.stderr.write("executing %s\n" % str(cmd))
226 retcode = subprocess.call(cmd, shell=expand)
227 if retcode and not ignore_error:
228 raise CalledProcessError(retcode, cmd)
230 return retcode
232 def p4_system(cmd):
233 """Specifically invoke p4 as the system command. """
234 real_cmd = p4_build_cmd(cmd)
235 expand = isinstance(real_cmd, basestring)
236 retcode = subprocess.call(real_cmd, shell=expand)
237 if retcode:
238 raise CalledProcessError(retcode, real_cmd)
240 _p4_version_string = None
241 def p4_version_string():
242 """Read the version string, showing just the last line, which
243 hopefully is the interesting version bit.
245 $ p4 -V
246 Perforce - The Fast Software Configuration Management System.
247 Copyright 1995-2011 Perforce Software. All rights reserved.
248 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
250 global _p4_version_string
251 if not _p4_version_string:
252 a = p4_read_pipe_lines(["-V"])
253 _p4_version_string = a[-1].rstrip()
254 return _p4_version_string
256 def p4_integrate(src, dest):
257 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
259 def p4_sync(f, *options):
260 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
262 def p4_add(f):
263 # forcibly add file names with wildcards
264 if wildcard_present(f):
265 p4_system(["add", "-f", f])
266 else:
267 p4_system(["add", f])
269 def p4_delete(f):
270 p4_system(["delete", wildcard_encode(f)])
272 def p4_edit(f, *options):
273 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
275 def p4_revert(f):
276 p4_system(["revert", wildcard_encode(f)])
278 def p4_reopen(type, f):
279 p4_system(["reopen", "-t", type, wildcard_encode(f)])
281 def p4_reopen_in_change(changelist, files):
282 cmd = ["reopen", "-c", str(changelist)] + files
283 p4_system(cmd)
285 def p4_move(src, dest):
286 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
288 def p4_last_change():
289 results = p4CmdList(["changes", "-m", "1"])
290 return int(results[0]['change'])
292 def p4_describe(change):
293 """Make sure it returns a valid result by checking for
294 the presence of field "time". Return a dict of the
295 results."""
297 ds = p4CmdList(["describe", "-s", str(change)])
298 if len(ds) != 1:
299 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
301 d = ds[0]
303 if "p4ExitCode" in d:
304 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
305 str(d)))
306 if "code" in d:
307 if d["code"] == "error":
308 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
310 if "time" not in d:
311 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
313 return d
316 # Canonicalize the p4 type and return a tuple of the
317 # base type, plus any modifiers. See "p4 help filetypes"
318 # for a list and explanation.
320 def split_p4_type(p4type):
322 p4_filetypes_historical = {
323 "ctempobj": "binary+Sw",
324 "ctext": "text+C",
325 "cxtext": "text+Cx",
326 "ktext": "text+k",
327 "kxtext": "text+kx",
328 "ltext": "text+F",
329 "tempobj": "binary+FSw",
330 "ubinary": "binary+F",
331 "uresource": "resource+F",
332 "uxbinary": "binary+Fx",
333 "xbinary": "binary+x",
334 "xltext": "text+Fx",
335 "xtempobj": "binary+Swx",
336 "xtext": "text+x",
337 "xunicode": "unicode+x",
338 "xutf16": "utf16+x",
340 if p4type in p4_filetypes_historical:
341 p4type = p4_filetypes_historical[p4type]
342 mods = ""
343 s = p4type.split("+")
344 base = s[0]
345 mods = ""
346 if len(s) > 1:
347 mods = s[1]
348 return (base, mods)
351 # return the raw p4 type of a file (text, text+ko, etc)
353 def p4_type(f):
354 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
355 return results[0]['headType']
358 # Given a type base and modifier, return a regexp matching
359 # the keywords that can be expanded in the file
361 def p4_keywords_regexp_for_type(base, type_mods):
362 if base in ("text", "unicode", "binary"):
363 kwords = None
364 if "ko" in type_mods:
365 kwords = 'Id|Header'
366 elif "k" in type_mods:
367 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
368 else:
369 return None
370 pattern = r"""
371 \$ # Starts with a dollar, followed by...
372 (%s) # one of the keywords, followed by...
373 (:[^$\n]+)? # possibly an old expansion, followed by...
374 \$ # another dollar
375 """ % kwords
376 return pattern
377 else:
378 return None
381 # Given a file, return a regexp matching the possible
382 # RCS keywords that will be expanded, or None for files
383 # with kw expansion turned off.
385 def p4_keywords_regexp_for_file(file):
386 if not os.path.exists(file):
387 return None
388 else:
389 (type_base, type_mods) = split_p4_type(p4_type(file))
390 return p4_keywords_regexp_for_type(type_base, type_mods)
392 def setP4ExecBit(file, mode):
393 # Reopens an already open file and changes the execute bit to match
394 # the execute bit setting in the passed in mode.
396 p4Type = "+x"
398 if not isModeExec(mode):
399 p4Type = getP4OpenedType(file)
400 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
401 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
402 if p4Type[-1] == "+":
403 p4Type = p4Type[0:-1]
405 p4_reopen(p4Type, file)
407 def getP4OpenedType(file):
408 # Returns the perforce file type for the given file.
410 result = p4_read_pipe(["opened", wildcard_encode(file)])
411 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
412 if match:
413 return match.group(1)
414 else:
415 die("Could not determine file type for %s (result: '%s')" % (file, result))
417 # Return the set of all p4 labels
418 def getP4Labels(depotPaths):
419 labels = set()
420 if isinstance(depotPaths,basestring):
421 depotPaths = [depotPaths]
423 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
424 label = l['label']
425 labels.add(label)
427 return labels
429 # Return the set of all git tags
430 def getGitTags():
431 gitTags = set()
432 for line in read_pipe_lines(["git", "tag"]):
433 tag = line.strip()
434 gitTags.add(tag)
435 return gitTags
437 def diffTreePattern():
438 # This is a simple generator for the diff tree regex pattern. This could be
439 # a class variable if this and parseDiffTreeEntry were a part of a class.
440 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
441 while True:
442 yield pattern
444 def parseDiffTreeEntry(entry):
445 """Parses a single diff tree entry into its component elements.
447 See git-diff-tree(1) manpage for details about the format of the diff
448 output. This method returns a dictionary with the following elements:
450 src_mode - The mode of the source file
451 dst_mode - The mode of the destination file
452 src_sha1 - The sha1 for the source file
453 dst_sha1 - The sha1 fr the destination file
454 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
455 status_score - The score for the status (applicable for 'C' and 'R'
456 statuses). This is None if there is no score.
457 src - The path for the source file.
458 dst - The path for the destination file. This is only present for
459 copy or renames. If it is not present, this is None.
461 If the pattern is not matched, None is returned."""
463 match = diffTreePattern().next().match(entry)
464 if match:
465 return {
466 'src_mode': match.group(1),
467 'dst_mode': match.group(2),
468 'src_sha1': match.group(3),
469 'dst_sha1': match.group(4),
470 'status': match.group(5),
471 'status_score': match.group(6),
472 'src': match.group(7),
473 'dst': match.group(10)
475 return None
477 def isModeExec(mode):
478 # Returns True if the given git mode represents an executable file,
479 # otherwise False.
480 return mode[-3:] == "755"
482 def isModeExecChanged(src_mode, dst_mode):
483 return isModeExec(src_mode) != isModeExec(dst_mode)
485 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
487 if isinstance(cmd,basestring):
488 cmd = "-G " + cmd
489 expand = True
490 else:
491 cmd = ["-G"] + cmd
492 expand = False
494 cmd = p4_build_cmd(cmd)
495 if verbose:
496 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
498 # Use a temporary file to avoid deadlocks without
499 # subprocess.communicate(), which would put another copy
500 # of stdout into memory.
501 stdin_file = None
502 if stdin is not None:
503 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
504 if isinstance(stdin,basestring):
505 stdin_file.write(stdin)
506 else:
507 for i in stdin:
508 stdin_file.write(i + '\n')
509 stdin_file.flush()
510 stdin_file.seek(0)
512 p4 = subprocess.Popen(cmd,
513 shell=expand,
514 stdin=stdin_file,
515 stdout=subprocess.PIPE)
517 result = []
518 try:
519 while True:
520 entry = marshal.load(p4.stdout)
521 if cb is not None:
522 cb(entry)
523 else:
524 result.append(entry)
525 except EOFError:
526 pass
527 exitCode = p4.wait()
528 if exitCode != 0:
529 entry = {}
530 entry["p4ExitCode"] = exitCode
531 result.append(entry)
533 return result
535 def p4Cmd(cmd):
536 list = p4CmdList(cmd)
537 result = {}
538 for entry in list:
539 result.update(entry)
540 return result;
542 def p4Where(depotPath):
543 if not depotPath.endswith("/"):
544 depotPath += "/"
545 depotPathLong = depotPath + "..."
546 outputList = p4CmdList(["where", depotPathLong])
547 output = None
548 for entry in outputList:
549 if "depotFile" in entry:
550 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
551 # The base path always ends with "/...".
552 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
553 output = entry
554 break
555 elif "data" in entry:
556 data = entry.get("data")
557 space = data.find(" ")
558 if data[:space] == depotPath:
559 output = entry
560 break
561 if output == None:
562 return ""
563 if output["code"] == "error":
564 return ""
565 clientPath = ""
566 if "path" in output:
567 clientPath = output.get("path")
568 elif "data" in output:
569 data = output.get("data")
570 lastSpace = data.rfind(" ")
571 clientPath = data[lastSpace + 1:]
573 if clientPath.endswith("..."):
574 clientPath = clientPath[:-3]
575 return clientPath
577 def currentGitBranch():
578 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
579 if retcode != 0:
580 # on a detached head
581 return None
582 else:
583 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
585 def isValidGitDir(path):
586 return git_dir(path) != None
588 def parseRevision(ref):
589 return read_pipe("git rev-parse %s" % ref).strip()
591 def branchExists(ref):
592 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
593 ignore_error=True)
594 return len(rev) > 0
596 def extractLogMessageFromGitCommit(commit):
597 logMessage = ""
599 ## fixme: title is first line of commit, not 1st paragraph.
600 foundTitle = False
601 for log in read_pipe_lines("git cat-file commit %s" % commit):
602 if not foundTitle:
603 if len(log) == 1:
604 foundTitle = True
605 continue
607 logMessage += log
608 return logMessage
610 def extractSettingsGitLog(log):
611 values = {}
612 for line in log.split("\n"):
613 line = line.strip()
614 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
615 if not m:
616 continue
618 assignments = m.group(1).split (':')
619 for a in assignments:
620 vals = a.split ('=')
621 key = vals[0].strip()
622 val = ('='.join (vals[1:])).strip()
623 if val.endswith ('\"') and val.startswith('"'):
624 val = val[1:-1]
626 values[key] = val
628 paths = values.get("depot-paths")
629 if not paths:
630 paths = values.get("depot-path")
631 if paths:
632 values['depot-paths'] = paths.split(',')
633 return values
635 def gitBranchExists(branch):
636 proc = subprocess.Popen(["git", "rev-parse", branch],
637 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
638 return proc.wait() == 0;
640 _gitConfig = {}
642 def gitConfig(key, typeSpecifier=None):
643 if not _gitConfig.has_key(key):
644 cmd = [ "git", "config" ]
645 if typeSpecifier:
646 cmd += [ typeSpecifier ]
647 cmd += [ key ]
648 s = read_pipe(cmd, ignore_error=True)
649 _gitConfig[key] = s.strip()
650 return _gitConfig[key]
652 def gitConfigBool(key):
653 """Return a bool, using git config --bool. It is True only if the
654 variable is set to true, and False if set to false or not present
655 in the config."""
657 if not _gitConfig.has_key(key):
658 _gitConfig[key] = gitConfig(key, '--bool') == "true"
659 return _gitConfig[key]
661 def gitConfigInt(key):
662 if not _gitConfig.has_key(key):
663 cmd = [ "git", "config", "--int", key ]
664 s = read_pipe(cmd, ignore_error=True)
665 v = s.strip()
666 try:
667 _gitConfig[key] = int(gitConfig(key, '--int'))
668 except ValueError:
669 _gitConfig[key] = None
670 return _gitConfig[key]
672 def gitConfigList(key):
673 if not _gitConfig.has_key(key):
674 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
675 _gitConfig[key] = s.strip().split(os.linesep)
676 if _gitConfig[key] == ['']:
677 _gitConfig[key] = []
678 return _gitConfig[key]
680 def p4BranchesInGit(branchesAreInRemotes=True):
681 """Find all the branches whose names start with "p4/", looking
682 in remotes or heads as specified by the argument. Return
683 a dictionary of { branch: revision } for each one found.
684 The branch names are the short names, without any
685 "p4/" prefix."""
687 branches = {}
689 cmdline = "git rev-parse --symbolic "
690 if branchesAreInRemotes:
691 cmdline += "--remotes"
692 else:
693 cmdline += "--branches"
695 for line in read_pipe_lines(cmdline):
696 line = line.strip()
698 # only import to p4/
699 if not line.startswith('p4/'):
700 continue
701 # special symbolic ref to p4/master
702 if line == "p4/HEAD":
703 continue
705 # strip off p4/ prefix
706 branch = line[len("p4/"):]
708 branches[branch] = parseRevision(line)
710 return branches
712 def branch_exists(branch):
713 """Make sure that the given ref name really exists."""
715 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
716 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
717 out, _ = p.communicate()
718 if p.returncode:
719 return False
720 # expect exactly one line of output: the branch name
721 return out.rstrip() == branch
723 def findUpstreamBranchPoint(head = "HEAD"):
724 branches = p4BranchesInGit()
725 # map from depot-path to branch name
726 branchByDepotPath = {}
727 for branch in branches.keys():
728 tip = branches[branch]
729 log = extractLogMessageFromGitCommit(tip)
730 settings = extractSettingsGitLog(log)
731 if settings.has_key("depot-paths"):
732 paths = ",".join(settings["depot-paths"])
733 branchByDepotPath[paths] = "remotes/p4/" + branch
735 settings = None
736 parent = 0
737 while parent < 65535:
738 commit = head + "~%s" % parent
739 log = extractLogMessageFromGitCommit(commit)
740 settings = extractSettingsGitLog(log)
741 if settings.has_key("depot-paths"):
742 paths = ",".join(settings["depot-paths"])
743 if branchByDepotPath.has_key(paths):
744 return [branchByDepotPath[paths], settings]
746 parent = parent + 1
748 return ["", settings]
750 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
751 if not silent:
752 print ("Creating/updating branch(es) in %s based on origin branch(es)"
753 % localRefPrefix)
755 originPrefix = "origin/p4/"
757 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
758 line = line.strip()
759 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
760 continue
762 headName = line[len(originPrefix):]
763 remoteHead = localRefPrefix + headName
764 originHead = line
766 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
767 if (not original.has_key('depot-paths')
768 or not original.has_key('change')):
769 continue
771 update = False
772 if not gitBranchExists(remoteHead):
773 if verbose:
774 print "creating %s" % remoteHead
775 update = True
776 else:
777 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
778 if settings.has_key('change') > 0:
779 if settings['depot-paths'] == original['depot-paths']:
780 originP4Change = int(original['change'])
781 p4Change = int(settings['change'])
782 if originP4Change > p4Change:
783 print ("%s (%s) is newer than %s (%s). "
784 "Updating p4 branch from origin."
785 % (originHead, originP4Change,
786 remoteHead, p4Change))
787 update = True
788 else:
789 print ("Ignoring: %s was imported from %s while "
790 "%s was imported from %s"
791 % (originHead, ','.join(original['depot-paths']),
792 remoteHead, ','.join(settings['depot-paths'])))
794 if update:
795 system("git update-ref %s %s" % (remoteHead, originHead))
797 def originP4BranchesExist():
798 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
801 def p4ParseNumericChangeRange(parts):
802 changeStart = int(parts[0][1:])
803 if parts[1] == '#head':
804 changeEnd = p4_last_change()
805 else:
806 changeEnd = int(parts[1])
808 return (changeStart, changeEnd)
810 def chooseBlockSize(blockSize):
811 if blockSize:
812 return blockSize
813 else:
814 return defaultBlockSize
816 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
817 assert depotPaths
819 # Parse the change range into start and end. Try to find integer
820 # revision ranges as these can be broken up into blocks to avoid
821 # hitting server-side limits (maxrows, maxscanresults). But if
822 # that doesn't work, fall back to using the raw revision specifier
823 # strings, without using block mode.
825 if changeRange is None or changeRange == '':
826 changeStart = 1
827 changeEnd = p4_last_change()
828 block_size = chooseBlockSize(requestedBlockSize)
829 else:
830 parts = changeRange.split(',')
831 assert len(parts) == 2
832 try:
833 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
834 block_size = chooseBlockSize(requestedBlockSize)
835 except:
836 changeStart = parts[0][1:]
837 changeEnd = parts[1]
838 if requestedBlockSize:
839 die("cannot use --changes-block-size with non-numeric revisions")
840 block_size = None
842 changes = set()
844 # Retrieve changes a block at a time, to prevent running
845 # into a MaxResults/MaxScanRows error from the server.
847 while True:
848 cmd = ['changes']
850 if block_size:
851 end = min(changeEnd, changeStart + block_size)
852 revisionRange = "%d,%d" % (changeStart, end)
853 else:
854 revisionRange = "%s,%s" % (changeStart, changeEnd)
856 for p in depotPaths:
857 cmd += ["%s...@%s" % (p, revisionRange)]
859 # Insert changes in chronological order
860 for line in reversed(p4_read_pipe_lines(cmd)):
861 changes.add(int(line.split(" ")[1]))
863 if not block_size:
864 break
866 if end >= changeEnd:
867 break
869 changeStart = end + 1
871 changes = sorted(changes)
872 return changes
874 def p4PathStartsWith(path, prefix):
875 # This method tries to remedy a potential mixed-case issue:
877 # If UserA adds //depot/DirA/file1
878 # and UserB adds //depot/dira/file2
880 # we may or may not have a problem. If you have core.ignorecase=true,
881 # we treat DirA and dira as the same directory
882 if gitConfigBool("core.ignorecase"):
883 return path.lower().startswith(prefix.lower())
884 return path.startswith(prefix)
886 def getClientSpec():
887 """Look at the p4 client spec, create a View() object that contains
888 all the mappings, and return it."""
890 specList = p4CmdList("client -o")
891 if len(specList) != 1:
892 die('Output from "client -o" is %d lines, expecting 1' %
893 len(specList))
895 # dictionary of all client parameters
896 entry = specList[0]
898 # the //client/ name
899 client_name = entry["Client"]
901 # just the keys that start with "View"
902 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
904 # hold this new View
905 view = View(client_name)
907 # append the lines, in order, to the view
908 for view_num in range(len(view_keys)):
909 k = "View%d" % view_num
910 if k not in view_keys:
911 die("Expected view key %s missing" % k)
912 view.append(entry[k])
914 return view
916 def getClientRoot():
917 """Grab the client directory."""
919 output = p4CmdList("client -o")
920 if len(output) != 1:
921 die('Output from "client -o" is %d lines, expecting 1' % len(output))
923 entry = output[0]
924 if "Root" not in entry:
925 die('Client has no "Root"')
927 return entry["Root"]
930 # P4 wildcards are not allowed in filenames. P4 complains
931 # if you simply add them, but you can force it with "-f", in
932 # which case it translates them into %xx encoding internally.
934 def wildcard_decode(path):
935 # Search for and fix just these four characters. Do % last so
936 # that fixing it does not inadvertently create new %-escapes.
937 # Cannot have * in a filename in windows; untested as to
938 # what p4 would do in such a case.
939 if not platform.system() == "Windows":
940 path = path.replace("%2A", "*")
941 path = path.replace("%23", "#") \
942 .replace("%40", "@") \
943 .replace("%25", "%")
944 return path
946 def wildcard_encode(path):
947 # do % first to avoid double-encoding the %s introduced here
948 path = path.replace("%", "%25") \
949 .replace("*", "%2A") \
950 .replace("#", "%23") \
951 .replace("@", "%40")
952 return path
954 def wildcard_present(path):
955 m = re.search("[*#@%]", path)
956 return m is not None
958 class LargeFileSystem(object):
959 """Base class for large file system support."""
961 def __init__(self, writeToGitStream):
962 self.largeFiles = set()
963 self.writeToGitStream = writeToGitStream
965 def generatePointer(self, cloneDestination, contentFile):
966 """Return the content of a pointer file that is stored in Git instead of
967 the actual content."""
968 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
970 def pushFile(self, localLargeFile):
971 """Push the actual content which is not stored in the Git repository to
972 a server."""
973 assert False, "Method 'pushFile' required in " + self.__class__.__name__
975 def hasLargeFileExtension(self, relPath):
976 return reduce(
977 lambda a, b: a or b,
978 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
979 False
982 def generateTempFile(self, contents):
983 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
984 for d in contents:
985 contentFile.write(d)
986 contentFile.close()
987 return contentFile.name
989 def exceedsLargeFileThreshold(self, relPath, contents):
990 if gitConfigInt('git-p4.largeFileThreshold'):
991 contentsSize = sum(len(d) for d in contents)
992 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
993 return True
994 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
995 contentsSize = sum(len(d) for d in contents)
996 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
997 return False
998 contentTempFile = self.generateTempFile(contents)
999 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1000 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1001 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1002 zf.close()
1003 compressedContentsSize = zf.infolist()[0].compress_size
1004 os.remove(contentTempFile)
1005 os.remove(compressedContentFile.name)
1006 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1007 return True
1008 return False
1010 def addLargeFile(self, relPath):
1011 self.largeFiles.add(relPath)
1013 def removeLargeFile(self, relPath):
1014 self.largeFiles.remove(relPath)
1016 def isLargeFile(self, relPath):
1017 return relPath in self.largeFiles
1019 def processContent(self, git_mode, relPath, contents):
1020 """Processes the content of git fast import. This method decides if a
1021 file is stored in the large file system and handles all necessary
1022 steps."""
1023 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1024 contentTempFile = self.generateTempFile(contents)
1025 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1026 if pointer_git_mode:
1027 git_mode = pointer_git_mode
1028 if localLargeFile:
1029 # Move temp file to final location in large file system
1030 largeFileDir = os.path.dirname(localLargeFile)
1031 if not os.path.isdir(largeFileDir):
1032 os.makedirs(largeFileDir)
1033 shutil.move(contentTempFile, localLargeFile)
1034 self.addLargeFile(relPath)
1035 if gitConfigBool('git-p4.largeFilePush'):
1036 self.pushFile(localLargeFile)
1037 if verbose:
1038 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1039 return (git_mode, contents)
1041 class MockLFS(LargeFileSystem):
1042 """Mock large file system for testing."""
1044 def generatePointer(self, contentFile):
1045 """The pointer content is the original content prefixed with "pointer-".
1046 The local filename of the large file storage is derived from the file content.
1048 with open(contentFile, 'r') as f:
1049 content = next(f)
1050 gitMode = '100644'
1051 pointerContents = 'pointer-' + content
1052 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1053 return (gitMode, pointerContents, localLargeFile)
1055 def pushFile(self, localLargeFile):
1056 """The remote filename of the large file storage is the same as the local
1057 one but in a different directory.
1059 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1060 if not os.path.exists(remotePath):
1061 os.makedirs(remotePath)
1062 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1064 class GitLFS(LargeFileSystem):
1065 """Git LFS as backend for the git-p4 large file system.
1066 See https://git-lfs.github.com/ for details."""
1068 def __init__(self, *args):
1069 LargeFileSystem.__init__(self, *args)
1070 self.baseGitAttributes = []
1072 def generatePointer(self, contentFile):
1073 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1074 mode and content which is stored in the Git repository instead of
1075 the actual content. Return also the new location of the actual
1076 content.
1078 if os.path.getsize(contentFile) == 0:
1079 return (None, '', None)
1081 pointerProcess = subprocess.Popen(
1082 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1083 stdout=subprocess.PIPE
1085 pointerFile = pointerProcess.stdout.read()
1086 if pointerProcess.wait():
1087 os.remove(contentFile)
1088 die('git-lfs pointer command failed. Did you install the extension?')
1090 # Git LFS removed the preamble in the output of the 'pointer' command
1091 # starting from version 1.2.0. Check for the preamble here to support
1092 # earlier versions.
1093 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1094 if pointerFile.startswith('Git LFS pointer for'):
1095 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1097 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1098 localLargeFile = os.path.join(
1099 os.getcwd(),
1100 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1101 oid,
1103 # LFS Spec states that pointer files should not have the executable bit set.
1104 gitMode = '100644'
1105 return (gitMode, pointerFile, localLargeFile)
1107 def pushFile(self, localLargeFile):
1108 uploadProcess = subprocess.Popen(
1109 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1111 if uploadProcess.wait():
1112 die('git-lfs push command failed. Did you define a remote?')
1114 def generateGitAttributes(self):
1115 return (
1116 self.baseGitAttributes +
1118 '\n',
1119 '#\n',
1120 '# Git LFS (see https://git-lfs.github.com/)\n',
1121 '#\n',
1123 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1124 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1126 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1127 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1131 def addLargeFile(self, relPath):
1132 LargeFileSystem.addLargeFile(self, relPath)
1133 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1135 def removeLargeFile(self, relPath):
1136 LargeFileSystem.removeLargeFile(self, relPath)
1137 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1139 def processContent(self, git_mode, relPath, contents):
1140 if relPath == '.gitattributes':
1141 self.baseGitAttributes = contents
1142 return (git_mode, self.generateGitAttributes())
1143 else:
1144 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1146 class Command:
1147 def __init__(self):
1148 self.usage = "usage: %prog [options]"
1149 self.needsGit = True
1150 self.verbose = False
1152 class P4UserMap:
1153 def __init__(self):
1154 self.userMapFromPerforceServer = False
1155 self.myP4UserId = None
1157 def p4UserId(self):
1158 if self.myP4UserId:
1159 return self.myP4UserId
1161 results = p4CmdList("user -o")
1162 for r in results:
1163 if r.has_key('User'):
1164 self.myP4UserId = r['User']
1165 return r['User']
1166 die("Could not find your p4 user id")
1168 def p4UserIsMe(self, p4User):
1169 # return True if the given p4 user is actually me
1170 me = self.p4UserId()
1171 if not p4User or p4User != me:
1172 return False
1173 else:
1174 return True
1176 def getUserCacheFilename(self):
1177 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1178 return home + "/.gitp4-usercache.txt"
1180 def getUserMapFromPerforceServer(self):
1181 if self.userMapFromPerforceServer:
1182 return
1183 self.users = {}
1184 self.emails = {}
1186 for output in p4CmdList("users"):
1187 if not output.has_key("User"):
1188 continue
1189 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1190 self.emails[output["Email"]] = output["User"]
1192 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1193 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1194 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1195 if mapUser and len(mapUser[0]) == 3:
1196 user = mapUser[0][0]
1197 fullname = mapUser[0][1]
1198 email = mapUser[0][2]
1199 self.users[user] = fullname + " <" + email + ">"
1200 self.emails[email] = user
1202 s = ''
1203 for (key, val) in self.users.items():
1204 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1206 open(self.getUserCacheFilename(), "wb").write(s)
1207 self.userMapFromPerforceServer = True
1209 def loadUserMapFromCache(self):
1210 self.users = {}
1211 self.userMapFromPerforceServer = False
1212 try:
1213 cache = open(self.getUserCacheFilename(), "rb")
1214 lines = cache.readlines()
1215 cache.close()
1216 for line in lines:
1217 entry = line.strip().split("\t")
1218 self.users[entry[0]] = entry[1]
1219 except IOError:
1220 self.getUserMapFromPerforceServer()
1222 class P4Debug(Command):
1223 def __init__(self):
1224 Command.__init__(self)
1225 self.options = []
1226 self.description = "A tool to debug the output of p4 -G."
1227 self.needsGit = False
1229 def run(self, args):
1230 j = 0
1231 for output in p4CmdList(args):
1232 print 'Element: %d' % j
1233 j += 1
1234 print output
1235 return True
1237 class P4RollBack(Command):
1238 def __init__(self):
1239 Command.__init__(self)
1240 self.options = [
1241 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1243 self.description = "A tool to debug the multi-branch import. Don't use :)"
1244 self.rollbackLocalBranches = False
1246 def run(self, args):
1247 if len(args) != 1:
1248 return False
1249 maxChange = int(args[0])
1251 if "p4ExitCode" in p4Cmd("changes -m 1"):
1252 die("Problems executing p4");
1254 if self.rollbackLocalBranches:
1255 refPrefix = "refs/heads/"
1256 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1257 else:
1258 refPrefix = "refs/remotes/"
1259 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1261 for line in lines:
1262 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1263 line = line.strip()
1264 ref = refPrefix + line
1265 log = extractLogMessageFromGitCommit(ref)
1266 settings = extractSettingsGitLog(log)
1268 depotPaths = settings['depot-paths']
1269 change = settings['change']
1271 changed = False
1273 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1274 for p in depotPaths]))) == 0:
1275 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1276 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1277 continue
1279 while change and int(change) > maxChange:
1280 changed = True
1281 if self.verbose:
1282 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1283 system("git update-ref %s \"%s^\"" % (ref, ref))
1284 log = extractLogMessageFromGitCommit(ref)
1285 settings = extractSettingsGitLog(log)
1288 depotPaths = settings['depot-paths']
1289 change = settings['change']
1291 if changed:
1292 print "%s rewound to %s" % (ref, change)
1294 return True
1296 class P4Submit(Command, P4UserMap):
1298 conflict_behavior_choices = ("ask", "skip", "quit")
1300 def __init__(self):
1301 Command.__init__(self)
1302 P4UserMap.__init__(self)
1303 self.options = [
1304 optparse.make_option("--origin", dest="origin"),
1305 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1306 # preserve the user, requires relevant p4 permissions
1307 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1308 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1309 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1310 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1311 optparse.make_option("--conflict", dest="conflict_behavior",
1312 choices=self.conflict_behavior_choices),
1313 optparse.make_option("--branch", dest="branch"),
1314 optparse.make_option("--shelve", dest="shelve", action="store_true",
1315 help="Shelve instead of submit. Shelved files are reverted, "
1316 "restoring the workspace to the state before the shelve"),
1317 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1318 metavar="CHANGELIST",
1319 help="update an existing shelved changelist, implies --shelve")
1321 self.description = "Submit changes from git to the perforce depot."
1322 self.usage += " [name of git branch to submit into perforce depot]"
1323 self.origin = ""
1324 self.detectRenames = False
1325 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1326 self.dry_run = False
1327 self.shelve = False
1328 self.update_shelve = None
1329 self.prepare_p4_only = False
1330 self.conflict_behavior = None
1331 self.isWindows = (platform.system() == "Windows")
1332 self.exportLabels = False
1333 self.p4HasMoveCommand = p4_has_move_command()
1334 self.branch = None
1336 if gitConfig('git-p4.largeFileSystem'):
1337 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1339 def check(self):
1340 if len(p4CmdList("opened ...")) > 0:
1341 die("You have files opened with perforce! Close them before starting the sync.")
1343 def separate_jobs_from_description(self, message):
1344 """Extract and return a possible Jobs field in the commit
1345 message. It goes into a separate section in the p4 change
1346 specification.
1348 A jobs line starts with "Jobs:" and looks like a new field
1349 in a form. Values are white-space separated on the same
1350 line or on following lines that start with a tab.
1352 This does not parse and extract the full git commit message
1353 like a p4 form. It just sees the Jobs: line as a marker
1354 to pass everything from then on directly into the p4 form,
1355 but outside the description section.
1357 Return a tuple (stripped log message, jobs string)."""
1359 m = re.search(r'^Jobs:', message, re.MULTILINE)
1360 if m is None:
1361 return (message, None)
1363 jobtext = message[m.start():]
1364 stripped_message = message[:m.start()].rstrip()
1365 return (stripped_message, jobtext)
1367 def prepareLogMessage(self, template, message, jobs):
1368 """Edits the template returned from "p4 change -o" to insert
1369 the message in the Description field, and the jobs text in
1370 the Jobs field."""
1371 result = ""
1373 inDescriptionSection = False
1375 for line in template.split("\n"):
1376 if line.startswith("#"):
1377 result += line + "\n"
1378 continue
1380 if inDescriptionSection:
1381 if line.startswith("Files:") or line.startswith("Jobs:"):
1382 inDescriptionSection = False
1383 # insert Jobs section
1384 if jobs:
1385 result += jobs + "\n"
1386 else:
1387 continue
1388 else:
1389 if line.startswith("Description:"):
1390 inDescriptionSection = True
1391 line += "\n"
1392 for messageLine in message.split("\n"):
1393 line += "\t" + messageLine + "\n"
1395 result += line + "\n"
1397 return result
1399 def patchRCSKeywords(self, file, pattern):
1400 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1401 (handle, outFileName) = tempfile.mkstemp(dir='.')
1402 try:
1403 outFile = os.fdopen(handle, "w+")
1404 inFile = open(file, "r")
1405 regexp = re.compile(pattern, re.VERBOSE)
1406 for line in inFile.readlines():
1407 line = regexp.sub(r'$\1$', line)
1408 outFile.write(line)
1409 inFile.close()
1410 outFile.close()
1411 # Forcibly overwrite the original file
1412 os.unlink(file)
1413 shutil.move(outFileName, file)
1414 except:
1415 # cleanup our temporary file
1416 os.unlink(outFileName)
1417 print "Failed to strip RCS keywords in %s" % file
1418 raise
1420 print "Patched up RCS keywords in %s" % file
1422 def p4UserForCommit(self,id):
1423 # Return the tuple (perforce user,git email) for a given git commit id
1424 self.getUserMapFromPerforceServer()
1425 gitEmail = read_pipe(["git", "log", "--max-count=1",
1426 "--format=%ae", id])
1427 gitEmail = gitEmail.strip()
1428 if not self.emails.has_key(gitEmail):
1429 return (None,gitEmail)
1430 else:
1431 return (self.emails[gitEmail],gitEmail)
1433 def checkValidP4Users(self,commits):
1434 # check if any git authors cannot be mapped to p4 users
1435 for id in commits:
1436 (user,email) = self.p4UserForCommit(id)
1437 if not user:
1438 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1439 if gitConfigBool("git-p4.allowMissingP4Users"):
1440 print "%s" % msg
1441 else:
1442 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1444 def lastP4Changelist(self):
1445 # Get back the last changelist number submitted in this client spec. This
1446 # then gets used to patch up the username in the change. If the same
1447 # client spec is being used by multiple processes then this might go
1448 # wrong.
1449 results = p4CmdList("client -o") # find the current client
1450 client = None
1451 for r in results:
1452 if r.has_key('Client'):
1453 client = r['Client']
1454 break
1455 if not client:
1456 die("could not get client spec")
1457 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1458 for r in results:
1459 if r.has_key('change'):
1460 return r['change']
1461 die("Could not get changelist number for last submit - cannot patch up user details")
1463 def modifyChangelistUser(self, changelist, newUser):
1464 # fixup the user field of a changelist after it has been submitted.
1465 changes = p4CmdList("change -o %s" % changelist)
1466 if len(changes) != 1:
1467 die("Bad output from p4 change modifying %s to user %s" %
1468 (changelist, newUser))
1470 c = changes[0]
1471 if c['User'] == newUser: return # nothing to do
1472 c['User'] = newUser
1473 input = marshal.dumps(c)
1475 result = p4CmdList("change -f -i", stdin=input)
1476 for r in result:
1477 if r.has_key('code'):
1478 if r['code'] == 'error':
1479 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1480 if r.has_key('data'):
1481 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1482 return
1483 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1485 def canChangeChangelists(self):
1486 # check to see if we have p4 admin or super-user permissions, either of
1487 # which are required to modify changelists.
1488 results = p4CmdList(["protects", self.depotPath])
1489 for r in results:
1490 if r.has_key('perm'):
1491 if r['perm'] == 'admin':
1492 return 1
1493 if r['perm'] == 'super':
1494 return 1
1495 return 0
1497 def prepareSubmitTemplate(self, changelist=None):
1498 """Run "p4 change -o" to grab a change specification template.
1499 This does not use "p4 -G", as it is nice to keep the submission
1500 template in original order, since a human might edit it.
1502 Remove lines in the Files section that show changes to files
1503 outside the depot path we're committing into."""
1505 [upstream, settings] = findUpstreamBranchPoint()
1507 template = ""
1508 inFilesSection = False
1509 args = ['change', '-o']
1510 if changelist:
1511 args.append(str(changelist))
1513 for line in p4_read_pipe_lines(args):
1514 if line.endswith("\r\n"):
1515 line = line[:-2] + "\n"
1516 if inFilesSection:
1517 if line.startswith("\t"):
1518 # path starts and ends with a tab
1519 path = line[1:]
1520 lastTab = path.rfind("\t")
1521 if lastTab != -1:
1522 path = path[:lastTab]
1523 if settings.has_key('depot-paths'):
1524 if not [p for p in settings['depot-paths']
1525 if p4PathStartsWith(path, p)]:
1526 continue
1527 else:
1528 if not p4PathStartsWith(path, self.depotPath):
1529 continue
1530 else:
1531 inFilesSection = False
1532 else:
1533 if line.startswith("Files:"):
1534 inFilesSection = True
1536 template += line
1538 return template
1540 def edit_template(self, template_file):
1541 """Invoke the editor to let the user change the submission
1542 message. Return true if okay to continue with the submit."""
1544 # if configured to skip the editing part, just submit
1545 if gitConfigBool("git-p4.skipSubmitEdit"):
1546 return True
1548 # look at the modification time, to check later if the user saved
1549 # the file
1550 mtime = os.stat(template_file).st_mtime
1552 # invoke the editor
1553 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1554 editor = os.environ.get("P4EDITOR")
1555 else:
1556 editor = read_pipe("git var GIT_EDITOR").strip()
1557 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1559 # If the file was not saved, prompt to see if this patch should
1560 # be skipped. But skip this verification step if configured so.
1561 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1562 return True
1564 # modification time updated means user saved the file
1565 if os.stat(template_file).st_mtime > mtime:
1566 return True
1568 while True:
1569 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1570 if response == 'y':
1571 return True
1572 if response == 'n':
1573 return False
1575 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1576 # diff
1577 if os.environ.has_key("P4DIFF"):
1578 del(os.environ["P4DIFF"])
1579 diff = ""
1580 for editedFile in editedFiles:
1581 diff += p4_read_pipe(['diff', '-du',
1582 wildcard_encode(editedFile)])
1584 # new file diff
1585 newdiff = ""
1586 for newFile in filesToAdd:
1587 newdiff += "==== new file ====\n"
1588 newdiff += "--- /dev/null\n"
1589 newdiff += "+++ %s\n" % newFile
1591 is_link = os.path.islink(newFile)
1592 expect_link = newFile in symlinks
1594 if is_link and expect_link:
1595 newdiff += "+%s\n" % os.readlink(newFile)
1596 else:
1597 f = open(newFile, "r")
1598 for line in f.readlines():
1599 newdiff += "+" + line
1600 f.close()
1602 return (diff + newdiff).replace('\r\n', '\n')
1604 def applyCommit(self, id):
1605 """Apply one commit, return True if it succeeded."""
1607 print "Applying", read_pipe(["git", "show", "-s",
1608 "--format=format:%h %s", id])
1610 (p4User, gitEmail) = self.p4UserForCommit(id)
1612 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1613 filesToAdd = set()
1614 filesToChangeType = set()
1615 filesToDelete = set()
1616 editedFiles = set()
1617 pureRenameCopy = set()
1618 symlinks = set()
1619 filesToChangeExecBit = {}
1620 all_files = list()
1622 for line in diff:
1623 diff = parseDiffTreeEntry(line)
1624 modifier = diff['status']
1625 path = diff['src']
1626 all_files.append(path)
1628 if modifier == "M":
1629 p4_edit(path)
1630 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1631 filesToChangeExecBit[path] = diff['dst_mode']
1632 editedFiles.add(path)
1633 elif modifier == "A":
1634 filesToAdd.add(path)
1635 filesToChangeExecBit[path] = diff['dst_mode']
1636 if path in filesToDelete:
1637 filesToDelete.remove(path)
1639 dst_mode = int(diff['dst_mode'], 8)
1640 if dst_mode == 0120000:
1641 symlinks.add(path)
1643 elif modifier == "D":
1644 filesToDelete.add(path)
1645 if path in filesToAdd:
1646 filesToAdd.remove(path)
1647 elif modifier == "C":
1648 src, dest = diff['src'], diff['dst']
1649 p4_integrate(src, dest)
1650 pureRenameCopy.add(dest)
1651 if diff['src_sha1'] != diff['dst_sha1']:
1652 p4_edit(dest)
1653 pureRenameCopy.discard(dest)
1654 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1655 p4_edit(dest)
1656 pureRenameCopy.discard(dest)
1657 filesToChangeExecBit[dest] = diff['dst_mode']
1658 if self.isWindows:
1659 # turn off read-only attribute
1660 os.chmod(dest, stat.S_IWRITE)
1661 os.unlink(dest)
1662 editedFiles.add(dest)
1663 elif modifier == "R":
1664 src, dest = diff['src'], diff['dst']
1665 if self.p4HasMoveCommand:
1666 p4_edit(src) # src must be open before move
1667 p4_move(src, dest) # opens for (move/delete, move/add)
1668 else:
1669 p4_integrate(src, dest)
1670 if diff['src_sha1'] != diff['dst_sha1']:
1671 p4_edit(dest)
1672 else:
1673 pureRenameCopy.add(dest)
1674 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1675 if not self.p4HasMoveCommand:
1676 p4_edit(dest) # with move: already open, writable
1677 filesToChangeExecBit[dest] = diff['dst_mode']
1678 if not self.p4HasMoveCommand:
1679 if self.isWindows:
1680 os.chmod(dest, stat.S_IWRITE)
1681 os.unlink(dest)
1682 filesToDelete.add(src)
1683 editedFiles.add(dest)
1684 elif modifier == "T":
1685 filesToChangeType.add(path)
1686 else:
1687 die("unknown modifier %s for %s" % (modifier, path))
1689 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1690 patchcmd = diffcmd + " | git apply "
1691 tryPatchCmd = patchcmd + "--check -"
1692 applyPatchCmd = patchcmd + "--check --apply -"
1693 patch_succeeded = True
1695 if os.system(tryPatchCmd) != 0:
1696 fixed_rcs_keywords = False
1697 patch_succeeded = False
1698 print "Unfortunately applying the change failed!"
1700 # Patch failed, maybe it's just RCS keyword woes. Look through
1701 # the patch to see if that's possible.
1702 if gitConfigBool("git-p4.attemptRCSCleanup"):
1703 file = None
1704 pattern = None
1705 kwfiles = {}
1706 for file in editedFiles | filesToDelete:
1707 # did this file's delta contain RCS keywords?
1708 pattern = p4_keywords_regexp_for_file(file)
1710 if pattern:
1711 # this file is a possibility...look for RCS keywords.
1712 regexp = re.compile(pattern, re.VERBOSE)
1713 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1714 if regexp.search(line):
1715 if verbose:
1716 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1717 kwfiles[file] = pattern
1718 break
1720 for file in kwfiles:
1721 if verbose:
1722 print "zapping %s with %s" % (line,pattern)
1723 # File is being deleted, so not open in p4. Must
1724 # disable the read-only bit on windows.
1725 if self.isWindows and file not in editedFiles:
1726 os.chmod(file, stat.S_IWRITE)
1727 self.patchRCSKeywords(file, kwfiles[file])
1728 fixed_rcs_keywords = True
1730 if fixed_rcs_keywords:
1731 print "Retrying the patch with RCS keywords cleaned up"
1732 if os.system(tryPatchCmd) == 0:
1733 patch_succeeded = True
1735 if not patch_succeeded:
1736 for f in editedFiles:
1737 p4_revert(f)
1738 return False
1741 # Apply the patch for real, and do add/delete/+x handling.
1743 system(applyPatchCmd)
1745 for f in filesToChangeType:
1746 p4_edit(f, "-t", "auto")
1747 for f in filesToAdd:
1748 p4_add(f)
1749 for f in filesToDelete:
1750 p4_revert(f)
1751 p4_delete(f)
1753 # Set/clear executable bits
1754 for f in filesToChangeExecBit.keys():
1755 mode = filesToChangeExecBit[f]
1756 setP4ExecBit(f, mode)
1758 if self.update_shelve:
1759 print("all_files = %s" % str(all_files))
1760 p4_reopen_in_change(self.update_shelve, all_files)
1763 # Build p4 change description, starting with the contents
1764 # of the git commit message.
1766 logMessage = extractLogMessageFromGitCommit(id)
1767 logMessage = logMessage.strip()
1768 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1770 template = self.prepareSubmitTemplate(self.update_shelve)
1771 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1773 if self.preserveUser:
1774 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1776 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1777 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1778 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1779 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1781 separatorLine = "######## everything below this line is just the diff #######\n"
1782 if not self.prepare_p4_only:
1783 submitTemplate += separatorLine
1784 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1786 (handle, fileName) = tempfile.mkstemp()
1787 tmpFile = os.fdopen(handle, "w+b")
1788 if self.isWindows:
1789 submitTemplate = submitTemplate.replace("\n", "\r\n")
1790 tmpFile.write(submitTemplate)
1791 tmpFile.close()
1793 if self.prepare_p4_only:
1795 # Leave the p4 tree prepared, and the submit template around
1796 # and let the user decide what to do next
1798 print
1799 print "P4 workspace prepared for submission."
1800 print "To submit or revert, go to client workspace"
1801 print " " + self.clientPath
1802 print
1803 print "To submit, use \"p4 submit\" to write a new description,"
1804 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1805 " \"git p4\"." % fileName
1806 print "You can delete the file \"%s\" when finished." % fileName
1808 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1809 print "To preserve change ownership by user %s, you must\n" \
1810 "do \"p4 change -f <change>\" after submitting and\n" \
1811 "edit the User field."
1812 if pureRenameCopy:
1813 print "After submitting, renamed files must be re-synced."
1814 print "Invoke \"p4 sync -f\" on each of these files:"
1815 for f in pureRenameCopy:
1816 print " " + f
1818 print
1819 print "To revert the changes, use \"p4 revert ...\", and delete"
1820 print "the submit template file \"%s\"" % fileName
1821 if filesToAdd:
1822 print "Since the commit adds new files, they must be deleted:"
1823 for f in filesToAdd:
1824 print " " + f
1825 print
1826 return True
1829 # Let the user edit the change description, then submit it.
1831 submitted = False
1833 try:
1834 if self.edit_template(fileName):
1835 # read the edited message and submit
1836 tmpFile = open(fileName, "rb")
1837 message = tmpFile.read()
1838 tmpFile.close()
1839 if self.isWindows:
1840 message = message.replace("\r\n", "\n")
1841 submitTemplate = message[:message.index(separatorLine)]
1843 if self.update_shelve:
1844 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1845 elif self.shelve:
1846 p4_write_pipe(['shelve', '-i'], submitTemplate)
1847 else:
1848 p4_write_pipe(['submit', '-i'], submitTemplate)
1849 # The rename/copy happened by applying a patch that created a
1850 # new file. This leaves it writable, which confuses p4.
1851 for f in pureRenameCopy:
1852 p4_sync(f, "-f")
1854 if self.preserveUser:
1855 if p4User:
1856 # Get last changelist number. Cannot easily get it from
1857 # the submit command output as the output is
1858 # unmarshalled.
1859 changelist = self.lastP4Changelist()
1860 self.modifyChangelistUser(changelist, p4User)
1862 submitted = True
1864 finally:
1865 # skip this patch
1866 if not submitted or self.shelve:
1867 if self.shelve:
1868 print ("Reverting shelved files.")
1869 else:
1870 print ("Submission cancelled, undoing p4 changes.")
1871 for f in editedFiles | filesToDelete:
1872 p4_revert(f)
1873 for f in filesToAdd:
1874 p4_revert(f)
1875 os.remove(f)
1877 os.remove(fileName)
1878 return submitted
1880 # Export git tags as p4 labels. Create a p4 label and then tag
1881 # with that.
1882 def exportGitTags(self, gitTags):
1883 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1884 if len(validLabelRegexp) == 0:
1885 validLabelRegexp = defaultLabelRegexp
1886 m = re.compile(validLabelRegexp)
1888 for name in gitTags:
1890 if not m.match(name):
1891 if verbose:
1892 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1893 continue
1895 # Get the p4 commit this corresponds to
1896 logMessage = extractLogMessageFromGitCommit(name)
1897 values = extractSettingsGitLog(logMessage)
1899 if not values.has_key('change'):
1900 # a tag pointing to something not sent to p4; ignore
1901 if verbose:
1902 print "git tag %s does not give a p4 commit" % name
1903 continue
1904 else:
1905 changelist = values['change']
1907 # Get the tag details.
1908 inHeader = True
1909 isAnnotated = False
1910 body = []
1911 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1912 l = l.strip()
1913 if inHeader:
1914 if re.match(r'tag\s+', l):
1915 isAnnotated = True
1916 elif re.match(r'\s*$', l):
1917 inHeader = False
1918 continue
1919 else:
1920 body.append(l)
1922 if not isAnnotated:
1923 body = ["lightweight tag imported by git p4\n"]
1925 # Create the label - use the same view as the client spec we are using
1926 clientSpec = getClientSpec()
1928 labelTemplate = "Label: %s\n" % name
1929 labelTemplate += "Description:\n"
1930 for b in body:
1931 labelTemplate += "\t" + b + "\n"
1932 labelTemplate += "View:\n"
1933 for depot_side in clientSpec.mappings:
1934 labelTemplate += "\t%s\n" % depot_side
1936 if self.dry_run:
1937 print "Would create p4 label %s for tag" % name
1938 elif self.prepare_p4_only:
1939 print "Not creating p4 label %s for tag due to option" \
1940 " --prepare-p4-only" % name
1941 else:
1942 p4_write_pipe(["label", "-i"], labelTemplate)
1944 # Use the label
1945 p4_system(["tag", "-l", name] +
1946 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1948 if verbose:
1949 print "created p4 label for tag %s" % name
1951 def run(self, args):
1952 if len(args) == 0:
1953 self.master = currentGitBranch()
1954 elif len(args) == 1:
1955 self.master = args[0]
1956 if not branchExists(self.master):
1957 die("Branch %s does not exist" % self.master)
1958 else:
1959 return False
1961 if self.master:
1962 allowSubmit = gitConfig("git-p4.allowSubmit")
1963 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1964 die("%s is not in git-p4.allowSubmit" % self.master)
1966 [upstream, settings] = findUpstreamBranchPoint()
1967 self.depotPath = settings['depot-paths'][0]
1968 if len(self.origin) == 0:
1969 self.origin = upstream
1971 if self.update_shelve:
1972 self.shelve = True
1974 if self.preserveUser:
1975 if not self.canChangeChangelists():
1976 die("Cannot preserve user names without p4 super-user or admin permissions")
1978 # if not set from the command line, try the config file
1979 if self.conflict_behavior is None:
1980 val = gitConfig("git-p4.conflict")
1981 if val:
1982 if val not in self.conflict_behavior_choices:
1983 die("Invalid value '%s' for config git-p4.conflict" % val)
1984 else:
1985 val = "ask"
1986 self.conflict_behavior = val
1988 if self.verbose:
1989 print "Origin branch is " + self.origin
1991 if len(self.depotPath) == 0:
1992 print "Internal error: cannot locate perforce depot path from existing branches"
1993 sys.exit(128)
1995 self.useClientSpec = False
1996 if gitConfigBool("git-p4.useclientspec"):
1997 self.useClientSpec = True
1998 if self.useClientSpec:
1999 self.clientSpecDirs = getClientSpec()
2001 # Check for the existence of P4 branches
2002 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2004 if self.useClientSpec and not branchesDetected:
2005 # all files are relative to the client spec
2006 self.clientPath = getClientRoot()
2007 else:
2008 self.clientPath = p4Where(self.depotPath)
2010 if self.clientPath == "":
2011 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2013 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2014 self.oldWorkingDirectory = os.getcwd()
2016 # ensure the clientPath exists
2017 new_client_dir = False
2018 if not os.path.exists(self.clientPath):
2019 new_client_dir = True
2020 os.makedirs(self.clientPath)
2022 chdir(self.clientPath, is_client_path=True)
2023 if self.dry_run:
2024 print "Would synchronize p4 checkout in %s" % self.clientPath
2025 else:
2026 print "Synchronizing p4 checkout..."
2027 if new_client_dir:
2028 # old one was destroyed, and maybe nobody told p4
2029 p4_sync("...", "-f")
2030 else:
2031 p4_sync("...")
2032 self.check()
2034 commits = []
2035 if self.master:
2036 commitish = self.master
2037 else:
2038 commitish = 'HEAD'
2040 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2041 commits.append(line.strip())
2042 commits.reverse()
2044 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2045 self.checkAuthorship = False
2046 else:
2047 self.checkAuthorship = True
2049 if self.preserveUser:
2050 self.checkValidP4Users(commits)
2053 # Build up a set of options to be passed to diff when
2054 # submitting each commit to p4.
2056 if self.detectRenames:
2057 # command-line -M arg
2058 self.diffOpts = "-M"
2059 else:
2060 # If not explicitly set check the config variable
2061 detectRenames = gitConfig("git-p4.detectRenames")
2063 if detectRenames.lower() == "false" or detectRenames == "":
2064 self.diffOpts = ""
2065 elif detectRenames.lower() == "true":
2066 self.diffOpts = "-M"
2067 else:
2068 self.diffOpts = "-M%s" % detectRenames
2070 # no command-line arg for -C or --find-copies-harder, just
2071 # config variables
2072 detectCopies = gitConfig("git-p4.detectCopies")
2073 if detectCopies.lower() == "false" or detectCopies == "":
2074 pass
2075 elif detectCopies.lower() == "true":
2076 self.diffOpts += " -C"
2077 else:
2078 self.diffOpts += " -C%s" % detectCopies
2080 if gitConfigBool("git-p4.detectCopiesHarder"):
2081 self.diffOpts += " --find-copies-harder"
2084 # Apply the commits, one at a time. On failure, ask if should
2085 # continue to try the rest of the patches, or quit.
2087 if self.dry_run:
2088 print "Would apply"
2089 applied = []
2090 last = len(commits) - 1
2091 for i, commit in enumerate(commits):
2092 if self.dry_run:
2093 print " ", read_pipe(["git", "show", "-s",
2094 "--format=format:%h %s", commit])
2095 ok = True
2096 else:
2097 ok = self.applyCommit(commit)
2098 if ok:
2099 applied.append(commit)
2100 else:
2101 if self.prepare_p4_only and i < last:
2102 print "Processing only the first commit due to option" \
2103 " --prepare-p4-only"
2104 break
2105 if i < last:
2106 quit = False
2107 while True:
2108 # prompt for what to do, or use the option/variable
2109 if self.conflict_behavior == "ask":
2110 print "What do you want to do?"
2111 response = raw_input("[s]kip this commit but apply"
2112 " the rest, or [q]uit? ")
2113 if not response:
2114 continue
2115 elif self.conflict_behavior == "skip":
2116 response = "s"
2117 elif self.conflict_behavior == "quit":
2118 response = "q"
2119 else:
2120 die("Unknown conflict_behavior '%s'" %
2121 self.conflict_behavior)
2123 if response[0] == "s":
2124 print "Skipping this commit, but applying the rest"
2125 break
2126 if response[0] == "q":
2127 print "Quitting"
2128 quit = True
2129 break
2130 if quit:
2131 break
2133 chdir(self.oldWorkingDirectory)
2134 shelved_applied = "shelved" if self.shelve else "applied"
2135 if self.dry_run:
2136 pass
2137 elif self.prepare_p4_only:
2138 pass
2139 elif len(commits) == len(applied):
2140 print ("All commits {0}!".format(shelved_applied))
2142 sync = P4Sync()
2143 if self.branch:
2144 sync.branch = self.branch
2145 sync.run([])
2147 rebase = P4Rebase()
2148 rebase.rebase()
2150 else:
2151 if len(applied) == 0:
2152 print ("No commits {0}.".format(shelved_applied))
2153 else:
2154 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2155 for c in commits:
2156 if c in applied:
2157 star = "*"
2158 else:
2159 star = " "
2160 print star, read_pipe(["git", "show", "-s",
2161 "--format=format:%h %s", c])
2162 print "You will have to do 'git p4 sync' and rebase."
2164 if gitConfigBool("git-p4.exportLabels"):
2165 self.exportLabels = True
2167 if self.exportLabels:
2168 p4Labels = getP4Labels(self.depotPath)
2169 gitTags = getGitTags()
2171 missingGitTags = gitTags - p4Labels
2172 self.exportGitTags(missingGitTags)
2174 # exit with error unless everything applied perfectly
2175 if len(commits) != len(applied):
2176 sys.exit(1)
2178 return True
2180 class View(object):
2181 """Represent a p4 view ("p4 help views"), and map files in a
2182 repo according to the view."""
2184 def __init__(self, client_name):
2185 self.mappings = []
2186 self.client_prefix = "//%s/" % client_name
2187 # cache results of "p4 where" to lookup client file locations
2188 self.client_spec_path_cache = {}
2190 def append(self, view_line):
2191 """Parse a view line, splitting it into depot and client
2192 sides. Append to self.mappings, preserving order. This
2193 is only needed for tag creation."""
2195 # Split the view line into exactly two words. P4 enforces
2196 # structure on these lines that simplifies this quite a bit.
2198 # Either or both words may be double-quoted.
2199 # Single quotes do not matter.
2200 # Double-quote marks cannot occur inside the words.
2201 # A + or - prefix is also inside the quotes.
2202 # There are no quotes unless they contain a space.
2203 # The line is already white-space stripped.
2204 # The two words are separated by a single space.
2206 if view_line[0] == '"':
2207 # First word is double quoted. Find its end.
2208 close_quote_index = view_line.find('"', 1)
2209 if close_quote_index <= 0:
2210 die("No first-word closing quote found: %s" % view_line)
2211 depot_side = view_line[1:close_quote_index]
2212 # skip closing quote and space
2213 rhs_index = close_quote_index + 1 + 1
2214 else:
2215 space_index = view_line.find(" ")
2216 if space_index <= 0:
2217 die("No word-splitting space found: %s" % view_line)
2218 depot_side = view_line[0:space_index]
2219 rhs_index = space_index + 1
2221 # prefix + means overlay on previous mapping
2222 if depot_side.startswith("+"):
2223 depot_side = depot_side[1:]
2225 # prefix - means exclude this path, leave out of mappings
2226 exclude = False
2227 if depot_side.startswith("-"):
2228 exclude = True
2229 depot_side = depot_side[1:]
2231 if not exclude:
2232 self.mappings.append(depot_side)
2234 def convert_client_path(self, clientFile):
2235 # chop off //client/ part to make it relative
2236 if not clientFile.startswith(self.client_prefix):
2237 die("No prefix '%s' on clientFile '%s'" %
2238 (self.client_prefix, clientFile))
2239 return clientFile[len(self.client_prefix):]
2241 def update_client_spec_path_cache(self, files):
2242 """ Caching file paths by "p4 where" batch query """
2244 # List depot file paths exclude that already cached
2245 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2247 if len(fileArgs) == 0:
2248 return # All files in cache
2250 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2251 for res in where_result:
2252 if "code" in res and res["code"] == "error":
2253 # assume error is "... file(s) not in client view"
2254 continue
2255 if "clientFile" not in res:
2256 die("No clientFile in 'p4 where' output")
2257 if "unmap" in res:
2258 # it will list all of them, but only one not unmap-ped
2259 continue
2260 if gitConfigBool("core.ignorecase"):
2261 res['depotFile'] = res['depotFile'].lower()
2262 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2264 # not found files or unmap files set to ""
2265 for depotFile in fileArgs:
2266 if gitConfigBool("core.ignorecase"):
2267 depotFile = depotFile.lower()
2268 if depotFile not in self.client_spec_path_cache:
2269 self.client_spec_path_cache[depotFile] = ""
2271 def map_in_client(self, depot_path):
2272 """Return the relative location in the client where this
2273 depot file should live. Returns "" if the file should
2274 not be mapped in the client."""
2276 if gitConfigBool("core.ignorecase"):
2277 depot_path = depot_path.lower()
2279 if depot_path in self.client_spec_path_cache:
2280 return self.client_spec_path_cache[depot_path]
2282 die( "Error: %s is not found in client spec path" % depot_path )
2283 return ""
2285 class P4Sync(Command, P4UserMap):
2286 delete_actions = ( "delete", "move/delete", "purge" )
2288 def __init__(self):
2289 Command.__init__(self)
2290 P4UserMap.__init__(self)
2291 self.options = [
2292 optparse.make_option("--branch", dest="branch"),
2293 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2294 optparse.make_option("--changesfile", dest="changesFile"),
2295 optparse.make_option("--silent", dest="silent", action="store_true"),
2296 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2297 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2298 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2299 help="Import into refs/heads/ , not refs/remotes"),
2300 optparse.make_option("--max-changes", dest="maxChanges",
2301 help="Maximum number of changes to import"),
2302 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2303 help="Internal block size to use when iteratively calling p4 changes"),
2304 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2305 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2306 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2307 help="Only sync files that are included in the Perforce Client Spec"),
2308 optparse.make_option("-/", dest="cloneExclude",
2309 action="append", type="string",
2310 help="exclude depot path"),
2312 self.description = """Imports from Perforce into a git repository.\n
2313 example:
2314 //depot/my/project/ -- to import the current head
2315 //depot/my/project/@all -- to import everything
2316 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2318 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2320 self.usage += " //depot/path[@revRange]"
2321 self.silent = False
2322 self.createdBranches = set()
2323 self.committedChanges = set()
2324 self.branch = ""
2325 self.detectBranches = False
2326 self.detectLabels = False
2327 self.importLabels = False
2328 self.changesFile = ""
2329 self.syncWithOrigin = True
2330 self.importIntoRemotes = True
2331 self.maxChanges = ""
2332 self.changes_block_size = None
2333 self.keepRepoPath = False
2334 self.depotPaths = None
2335 self.p4BranchesInGit = []
2336 self.cloneExclude = []
2337 self.useClientSpec = False
2338 self.useClientSpec_from_options = False
2339 self.clientSpecDirs = None
2340 self.tempBranches = []
2341 self.tempBranchLocation = "refs/git-p4-tmp"
2342 self.largeFileSystem = None
2344 if gitConfig('git-p4.largeFileSystem'):
2345 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2346 self.largeFileSystem = largeFileSystemConstructor(
2347 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2350 if gitConfig("git-p4.syncFromOrigin") == "false":
2351 self.syncWithOrigin = False
2353 # This is required for the "append" cloneExclude action
2354 def ensure_value(self, attr, value):
2355 if not hasattr(self, attr) or getattr(self, attr) is None:
2356 setattr(self, attr, value)
2357 return getattr(self, attr)
2359 # Force a checkpoint in fast-import and wait for it to finish
2360 def checkpoint(self):
2361 self.gitStream.write("checkpoint\n\n")
2362 self.gitStream.write("progress checkpoint\n\n")
2363 out = self.gitOutput.readline()
2364 if self.verbose:
2365 print "checkpoint finished: " + out
2367 def extractFilesFromCommit(self, commit):
2368 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2369 for path in self.cloneExclude]
2370 files = []
2371 fnum = 0
2372 while commit.has_key("depotFile%s" % fnum):
2373 path = commit["depotFile%s" % fnum]
2375 if [p for p in self.cloneExclude
2376 if p4PathStartsWith(path, p)]:
2377 found = False
2378 else:
2379 found = [p for p in self.depotPaths
2380 if p4PathStartsWith(path, p)]
2381 if not found:
2382 fnum = fnum + 1
2383 continue
2385 file = {}
2386 file["path"] = path
2387 file["rev"] = commit["rev%s" % fnum]
2388 file["action"] = commit["action%s" % fnum]
2389 file["type"] = commit["type%s" % fnum]
2390 files.append(file)
2391 fnum = fnum + 1
2392 return files
2394 def extractJobsFromCommit(self, commit):
2395 jobs = []
2396 jnum = 0
2397 while commit.has_key("job%s" % jnum):
2398 job = commit["job%s" % jnum]
2399 jobs.append(job)
2400 jnum = jnum + 1
2401 return jobs
2403 def stripRepoPath(self, path, prefixes):
2404 """When streaming files, this is called to map a p4 depot path
2405 to where it should go in git. The prefixes are either
2406 self.depotPaths, or self.branchPrefixes in the case of
2407 branch detection."""
2409 if self.useClientSpec:
2410 # branch detection moves files up a level (the branch name)
2411 # from what client spec interpretation gives
2412 path = self.clientSpecDirs.map_in_client(path)
2413 if self.detectBranches:
2414 for b in self.knownBranches:
2415 if path.startswith(b + "/"):
2416 path = path[len(b)+1:]
2418 elif self.keepRepoPath:
2419 # Preserve everything in relative path name except leading
2420 # //depot/; just look at first prefix as they all should
2421 # be in the same depot.
2422 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2423 if p4PathStartsWith(path, depot):
2424 path = path[len(depot):]
2426 else:
2427 for p in prefixes:
2428 if p4PathStartsWith(path, p):
2429 path = path[len(p):]
2430 break
2432 path = wildcard_decode(path)
2433 return path
2435 def splitFilesIntoBranches(self, commit):
2436 """Look at each depotFile in the commit to figure out to what
2437 branch it belongs."""
2439 if self.clientSpecDirs:
2440 files = self.extractFilesFromCommit(commit)
2441 self.clientSpecDirs.update_client_spec_path_cache(files)
2443 branches = {}
2444 fnum = 0
2445 while commit.has_key("depotFile%s" % fnum):
2446 path = commit["depotFile%s" % fnum]
2447 found = [p for p in self.depotPaths
2448 if p4PathStartsWith(path, p)]
2449 if not found:
2450 fnum = fnum + 1
2451 continue
2453 file = {}
2454 file["path"] = path
2455 file["rev"] = commit["rev%s" % fnum]
2456 file["action"] = commit["action%s" % fnum]
2457 file["type"] = commit["type%s" % fnum]
2458 fnum = fnum + 1
2460 # start with the full relative path where this file would
2461 # go in a p4 client
2462 if self.useClientSpec:
2463 relPath = self.clientSpecDirs.map_in_client(path)
2464 else:
2465 relPath = self.stripRepoPath(path, self.depotPaths)
2467 for branch in self.knownBranches.keys():
2468 # add a trailing slash so that a commit into qt/4.2foo
2469 # doesn't end up in qt/4.2, e.g.
2470 if relPath.startswith(branch + "/"):
2471 if branch not in branches:
2472 branches[branch] = []
2473 branches[branch].append(file)
2474 break
2476 return branches
2478 def writeToGitStream(self, gitMode, relPath, contents):
2479 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2480 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2481 for d in contents:
2482 self.gitStream.write(d)
2483 self.gitStream.write('\n')
2485 # output one file from the P4 stream
2486 # - helper for streamP4Files
2488 def streamOneP4File(self, file, contents):
2489 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2490 if verbose:
2491 size = int(self.stream_file['fileSize'])
2492 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2493 sys.stdout.flush()
2495 (type_base, type_mods) = split_p4_type(file["type"])
2497 git_mode = "100644"
2498 if "x" in type_mods:
2499 git_mode = "100755"
2500 if type_base == "symlink":
2501 git_mode = "120000"
2502 # p4 print on a symlink sometimes contains "target\n";
2503 # if it does, remove the newline
2504 data = ''.join(contents)
2505 if not data:
2506 # Some version of p4 allowed creating a symlink that pointed
2507 # to nothing. This causes p4 errors when checking out such
2508 # a change, and errors here too. Work around it by ignoring
2509 # the bad symlink; hopefully a future change fixes it.
2510 print "\nIgnoring empty symlink in %s" % file['depotFile']
2511 return
2512 elif data[-1] == '\n':
2513 contents = [data[:-1]]
2514 else:
2515 contents = [data]
2517 if type_base == "utf16":
2518 # p4 delivers different text in the python output to -G
2519 # than it does when using "print -o", or normal p4 client
2520 # operations. utf16 is converted to ascii or utf8, perhaps.
2521 # But ascii text saved as -t utf16 is completely mangled.
2522 # Invoke print -o to get the real contents.
2524 # On windows, the newlines will always be mangled by print, so put
2525 # them back too. This is not needed to the cygwin windows version,
2526 # just the native "NT" type.
2528 try:
2529 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2530 except Exception as e:
2531 if 'Translation of file content failed' in str(e):
2532 type_base = 'binary'
2533 else:
2534 raise e
2535 else:
2536 if p4_version_string().find('/NT') >= 0:
2537 text = text.replace('\r\n', '\n')
2538 contents = [ text ]
2540 if type_base == "apple":
2541 # Apple filetype files will be streamed as a concatenation of
2542 # its appledouble header and the contents. This is useless
2543 # on both macs and non-macs. If using "print -q -o xx", it
2544 # will create "xx" with the data, and "%xx" with the header.
2545 # This is also not very useful.
2547 # Ideally, someday, this script can learn how to generate
2548 # appledouble files directly and import those to git, but
2549 # non-mac machines can never find a use for apple filetype.
2550 print "\nIgnoring apple filetype file %s" % file['depotFile']
2551 return
2553 # Note that we do not try to de-mangle keywords on utf16 files,
2554 # even though in theory somebody may want that.
2555 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2556 if pattern:
2557 regexp = re.compile(pattern, re.VERBOSE)
2558 text = ''.join(contents)
2559 text = regexp.sub(r'$\1$', text)
2560 contents = [ text ]
2562 try:
2563 relPath.decode('ascii')
2564 except:
2565 encoding = 'utf8'
2566 if gitConfig('git-p4.pathEncoding'):
2567 encoding = gitConfig('git-p4.pathEncoding')
2568 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2569 if self.verbose:
2570 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2572 if self.largeFileSystem:
2573 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2575 self.writeToGitStream(git_mode, relPath, contents)
2577 def streamOneP4Deletion(self, file):
2578 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2579 if verbose:
2580 sys.stdout.write("delete %s\n" % relPath)
2581 sys.stdout.flush()
2582 self.gitStream.write("D %s\n" % relPath)
2584 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2585 self.largeFileSystem.removeLargeFile(relPath)
2587 # handle another chunk of streaming data
2588 def streamP4FilesCb(self, marshalled):
2590 # catch p4 errors and complain
2591 err = None
2592 if "code" in marshalled:
2593 if marshalled["code"] == "error":
2594 if "data" in marshalled:
2595 err = marshalled["data"].rstrip()
2597 if not err and 'fileSize' in self.stream_file:
2598 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2599 if required_bytes > 0:
2600 err = 'Not enough space left on %s! Free at least %i MB.' % (
2601 os.getcwd(), required_bytes/1024/1024
2604 if err:
2605 f = None
2606 if self.stream_have_file_info:
2607 if "depotFile" in self.stream_file:
2608 f = self.stream_file["depotFile"]
2609 # force a failure in fast-import, else an empty
2610 # commit will be made
2611 self.gitStream.write("\n")
2612 self.gitStream.write("die-now\n")
2613 self.gitStream.close()
2614 # ignore errors, but make sure it exits first
2615 self.importProcess.wait()
2616 if f:
2617 die("Error from p4 print for %s: %s" % (f, err))
2618 else:
2619 die("Error from p4 print: %s" % err)
2621 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2622 # start of a new file - output the old one first
2623 self.streamOneP4File(self.stream_file, self.stream_contents)
2624 self.stream_file = {}
2625 self.stream_contents = []
2626 self.stream_have_file_info = False
2628 # pick up the new file information... for the
2629 # 'data' field we need to append to our array
2630 for k in marshalled.keys():
2631 if k == 'data':
2632 if 'streamContentSize' not in self.stream_file:
2633 self.stream_file['streamContentSize'] = 0
2634 self.stream_file['streamContentSize'] += len(marshalled['data'])
2635 self.stream_contents.append(marshalled['data'])
2636 else:
2637 self.stream_file[k] = marshalled[k]
2639 if (verbose and
2640 'streamContentSize' in self.stream_file and
2641 'fileSize' in self.stream_file and
2642 'depotFile' in self.stream_file):
2643 size = int(self.stream_file["fileSize"])
2644 if size > 0:
2645 progress = 100*self.stream_file['streamContentSize']/size
2646 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2647 sys.stdout.flush()
2649 self.stream_have_file_info = True
2651 # Stream directly from "p4 files" into "git fast-import"
2652 def streamP4Files(self, files):
2653 filesForCommit = []
2654 filesToRead = []
2655 filesToDelete = []
2657 for f in files:
2658 filesForCommit.append(f)
2659 if f['action'] in self.delete_actions:
2660 filesToDelete.append(f)
2661 else:
2662 filesToRead.append(f)
2664 # deleted files...
2665 for f in filesToDelete:
2666 self.streamOneP4Deletion(f)
2668 if len(filesToRead) > 0:
2669 self.stream_file = {}
2670 self.stream_contents = []
2671 self.stream_have_file_info = False
2673 # curry self argument
2674 def streamP4FilesCbSelf(entry):
2675 self.streamP4FilesCb(entry)
2677 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2679 p4CmdList(["-x", "-", "print"],
2680 stdin=fileArgs,
2681 cb=streamP4FilesCbSelf)
2683 # do the last chunk
2684 if self.stream_file.has_key('depotFile'):
2685 self.streamOneP4File(self.stream_file, self.stream_contents)
2687 def make_email(self, userid):
2688 if userid in self.users:
2689 return self.users[userid]
2690 else:
2691 return "%s <a@b>" % userid
2693 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2694 """ Stream a p4 tag.
2695 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2698 if verbose:
2699 print "writing tag %s for commit %s" % (labelName, commit)
2700 gitStream.write("tag %s\n" % labelName)
2701 gitStream.write("from %s\n" % commit)
2703 if labelDetails.has_key('Owner'):
2704 owner = labelDetails["Owner"]
2705 else:
2706 owner = None
2708 # Try to use the owner of the p4 label, or failing that,
2709 # the current p4 user id.
2710 if owner:
2711 email = self.make_email(owner)
2712 else:
2713 email = self.make_email(self.p4UserId())
2714 tagger = "%s %s %s" % (email, epoch, self.tz)
2716 gitStream.write("tagger %s\n" % tagger)
2718 print "labelDetails=",labelDetails
2719 if labelDetails.has_key('Description'):
2720 description = labelDetails['Description']
2721 else:
2722 description = 'Label from git p4'
2724 gitStream.write("data %d\n" % len(description))
2725 gitStream.write(description)
2726 gitStream.write("\n")
2728 def inClientSpec(self, path):
2729 if not self.clientSpecDirs:
2730 return True
2731 inClientSpec = self.clientSpecDirs.map_in_client(path)
2732 if not inClientSpec and self.verbose:
2733 print('Ignoring file outside of client spec: {0}'.format(path))
2734 return inClientSpec
2736 def hasBranchPrefix(self, path):
2737 if not self.branchPrefixes:
2738 return True
2739 hasPrefix = [p for p in self.branchPrefixes
2740 if p4PathStartsWith(path, p)]
2741 if not hasPrefix and self.verbose:
2742 print('Ignoring file outside of prefix: {0}'.format(path))
2743 return hasPrefix
2745 def commit(self, details, files, branch, parent = ""):
2746 epoch = details["time"]
2747 author = details["user"]
2748 jobs = self.extractJobsFromCommit(details)
2750 if self.verbose:
2751 print('commit into {0}'.format(branch))
2753 if self.clientSpecDirs:
2754 self.clientSpecDirs.update_client_spec_path_cache(files)
2756 files = [f for f in files
2757 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2759 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2760 print('Ignoring revision {0} as it would produce an empty commit.'
2761 .format(details['change']))
2762 return
2764 self.gitStream.write("commit %s\n" % branch)
2765 self.gitStream.write("mark :%s\n" % details["change"])
2766 self.committedChanges.add(int(details["change"]))
2767 committer = ""
2768 if author not in self.users:
2769 self.getUserMapFromPerforceServer()
2770 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2772 self.gitStream.write("committer %s\n" % committer)
2774 self.gitStream.write("data <<EOT\n")
2775 self.gitStream.write(details["desc"])
2776 if len(jobs) > 0:
2777 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2778 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2779 (','.join(self.branchPrefixes), details["change"]))
2780 if len(details['options']) > 0:
2781 self.gitStream.write(": options = %s" % details['options'])
2782 self.gitStream.write("]\nEOT\n\n")
2784 if len(parent) > 0:
2785 if self.verbose:
2786 print "parent %s" % parent
2787 self.gitStream.write("from %s\n" % parent)
2789 self.streamP4Files(files)
2790 self.gitStream.write("\n")
2792 change = int(details["change"])
2794 if self.labels.has_key(change):
2795 label = self.labels[change]
2796 labelDetails = label[0]
2797 labelRevisions = label[1]
2798 if self.verbose:
2799 print "Change %s is labelled %s" % (change, labelDetails)
2801 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2802 for p in self.branchPrefixes])
2804 if len(files) == len(labelRevisions):
2806 cleanedFiles = {}
2807 for info in files:
2808 if info["action"] in self.delete_actions:
2809 continue
2810 cleanedFiles[info["depotFile"]] = info["rev"]
2812 if cleanedFiles == labelRevisions:
2813 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2815 else:
2816 if not self.silent:
2817 print ("Tag %s does not match with change %s: files do not match."
2818 % (labelDetails["label"], change))
2820 else:
2821 if not self.silent:
2822 print ("Tag %s does not match with change %s: file count is different."
2823 % (labelDetails["label"], change))
2825 # Build a dictionary of changelists and labels, for "detect-labels" option.
2826 def getLabels(self):
2827 self.labels = {}
2829 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2830 if len(l) > 0 and not self.silent:
2831 print "Finding files belonging to labels in %s" % `self.depotPaths`
2833 for output in l:
2834 label = output["label"]
2835 revisions = {}
2836 newestChange = 0
2837 if self.verbose:
2838 print "Querying files for label %s" % label
2839 for file in p4CmdList(["files"] +
2840 ["%s...@%s" % (p, label)
2841 for p in self.depotPaths]):
2842 revisions[file["depotFile"]] = file["rev"]
2843 change = int(file["change"])
2844 if change > newestChange:
2845 newestChange = change
2847 self.labels[newestChange] = [output, revisions]
2849 if self.verbose:
2850 print "Label changes: %s" % self.labels.keys()
2852 # Import p4 labels as git tags. A direct mapping does not
2853 # exist, so assume that if all the files are at the same revision
2854 # then we can use that, or it's something more complicated we should
2855 # just ignore.
2856 def importP4Labels(self, stream, p4Labels):
2857 if verbose:
2858 print "import p4 labels: " + ' '.join(p4Labels)
2860 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2861 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2862 if len(validLabelRegexp) == 0:
2863 validLabelRegexp = defaultLabelRegexp
2864 m = re.compile(validLabelRegexp)
2866 for name in p4Labels:
2867 commitFound = False
2869 if not m.match(name):
2870 if verbose:
2871 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2872 continue
2874 if name in ignoredP4Labels:
2875 continue
2877 labelDetails = p4CmdList(['label', "-o", name])[0]
2879 # get the most recent changelist for each file in this label
2880 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2881 for p in self.depotPaths])
2883 if change.has_key('change'):
2884 # find the corresponding git commit; take the oldest commit
2885 changelist = int(change['change'])
2886 if changelist in self.committedChanges:
2887 gitCommit = ":%d" % changelist # use a fast-import mark
2888 commitFound = True
2889 else:
2890 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2891 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2892 if len(gitCommit) == 0:
2893 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2894 else:
2895 commitFound = True
2896 gitCommit = gitCommit.strip()
2898 if commitFound:
2899 # Convert from p4 time format
2900 try:
2901 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2902 except ValueError:
2903 print "Could not convert label time %s" % labelDetails['Update']
2904 tmwhen = 1
2906 when = int(time.mktime(tmwhen))
2907 self.streamTag(stream, name, labelDetails, gitCommit, when)
2908 if verbose:
2909 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2910 else:
2911 if verbose:
2912 print "Label %s has no changelists - possibly deleted?" % name
2914 if not commitFound:
2915 # We can't import this label; don't try again as it will get very
2916 # expensive repeatedly fetching all the files for labels that will
2917 # never be imported. If the label is moved in the future, the
2918 # ignore will need to be removed manually.
2919 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2921 def guessProjectName(self):
2922 for p in self.depotPaths:
2923 if p.endswith("/"):
2924 p = p[:-1]
2925 p = p[p.strip().rfind("/") + 1:]
2926 if not p.endswith("/"):
2927 p += "/"
2928 return p
2930 def getBranchMapping(self):
2931 lostAndFoundBranches = set()
2933 user = gitConfig("git-p4.branchUser")
2934 if len(user) > 0:
2935 command = "branches -u %s" % user
2936 else:
2937 command = "branches"
2939 for info in p4CmdList(command):
2940 details = p4Cmd(["branch", "-o", info["branch"]])
2941 viewIdx = 0
2942 while details.has_key("View%s" % viewIdx):
2943 paths = details["View%s" % viewIdx].split(" ")
2944 viewIdx = viewIdx + 1
2945 # require standard //depot/foo/... //depot/bar/... mapping
2946 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2947 continue
2948 source = paths[0]
2949 destination = paths[1]
2950 ## HACK
2951 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2952 source = source[len(self.depotPaths[0]):-4]
2953 destination = destination[len(self.depotPaths[0]):-4]
2955 if destination in self.knownBranches:
2956 if not self.silent:
2957 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2958 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2959 continue
2961 self.knownBranches[destination] = source
2963 lostAndFoundBranches.discard(destination)
2965 if source not in self.knownBranches:
2966 lostAndFoundBranches.add(source)
2968 # Perforce does not strictly require branches to be defined, so we also
2969 # check git config for a branch list.
2971 # Example of branch definition in git config file:
2972 # [git-p4]
2973 # branchList=main:branchA
2974 # branchList=main:branchB
2975 # branchList=branchA:branchC
2976 configBranches = gitConfigList("git-p4.branchList")
2977 for branch in configBranches:
2978 if branch:
2979 (source, destination) = branch.split(":")
2980 self.knownBranches[destination] = source
2982 lostAndFoundBranches.discard(destination)
2984 if source not in self.knownBranches:
2985 lostAndFoundBranches.add(source)
2988 for branch in lostAndFoundBranches:
2989 self.knownBranches[branch] = branch
2991 def getBranchMappingFromGitBranches(self):
2992 branches = p4BranchesInGit(self.importIntoRemotes)
2993 for branch in branches.keys():
2994 if branch == "master":
2995 branch = "main"
2996 else:
2997 branch = branch[len(self.projectName):]
2998 self.knownBranches[branch] = branch
3000 def updateOptionDict(self, d):
3001 option_keys = {}
3002 if self.keepRepoPath:
3003 option_keys['keepRepoPath'] = 1
3005 d["options"] = ' '.join(sorted(option_keys.keys()))
3007 def readOptions(self, d):
3008 self.keepRepoPath = (d.has_key('options')
3009 and ('keepRepoPath' in d['options']))
3011 def gitRefForBranch(self, branch):
3012 if branch == "main":
3013 return self.refPrefix + "master"
3015 if len(branch) <= 0:
3016 return branch
3018 return self.refPrefix + self.projectName + branch
3020 def gitCommitByP4Change(self, ref, change):
3021 if self.verbose:
3022 print "looking in ref " + ref + " for change %s using bisect..." % change
3024 earliestCommit = ""
3025 latestCommit = parseRevision(ref)
3027 while True:
3028 if self.verbose:
3029 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3030 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3031 if len(next) == 0:
3032 if self.verbose:
3033 print "argh"
3034 return ""
3035 log = extractLogMessageFromGitCommit(next)
3036 settings = extractSettingsGitLog(log)
3037 currentChange = int(settings['change'])
3038 if self.verbose:
3039 print "current change %s" % currentChange
3041 if currentChange == change:
3042 if self.verbose:
3043 print "found %s" % next
3044 return next
3046 if currentChange < change:
3047 earliestCommit = "^%s" % next
3048 else:
3049 latestCommit = "%s" % next
3051 return ""
3053 def importNewBranch(self, branch, maxChange):
3054 # make fast-import flush all changes to disk and update the refs using the checkpoint
3055 # command so that we can try to find the branch parent in the git history
3056 self.gitStream.write("checkpoint\n\n");
3057 self.gitStream.flush();
3058 branchPrefix = self.depotPaths[0] + branch + "/"
3059 range = "@1,%s" % maxChange
3060 #print "prefix" + branchPrefix
3061 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3062 if len(changes) <= 0:
3063 return False
3064 firstChange = changes[0]
3065 #print "first change in branch: %s" % firstChange
3066 sourceBranch = self.knownBranches[branch]
3067 sourceDepotPath = self.depotPaths[0] + sourceBranch
3068 sourceRef = self.gitRefForBranch(sourceBranch)
3069 #print "source " + sourceBranch
3071 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3072 #print "branch parent: %s" % branchParentChange
3073 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3074 if len(gitParent) > 0:
3075 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3076 #print "parent git commit: %s" % gitParent
3078 self.importChanges(changes)
3079 return True
3081 def searchParent(self, parent, branch, target):
3082 parentFound = False
3083 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3084 "--no-merges", parent]):
3085 blob = blob.strip()
3086 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3087 parentFound = True
3088 if self.verbose:
3089 print "Found parent of %s in commit %s" % (branch, blob)
3090 break
3091 if parentFound:
3092 return blob
3093 else:
3094 return None
3096 def importChanges(self, changes):
3097 cnt = 1
3098 for change in changes:
3099 description = p4_describe(change)
3100 self.updateOptionDict(description)
3102 if not self.silent:
3103 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3104 sys.stdout.flush()
3105 cnt = cnt + 1
3107 try:
3108 if self.detectBranches:
3109 branches = self.splitFilesIntoBranches(description)
3110 for branch in branches.keys():
3111 ## HACK --hwn
3112 branchPrefix = self.depotPaths[0] + branch + "/"
3113 self.branchPrefixes = [ branchPrefix ]
3115 parent = ""
3117 filesForCommit = branches[branch]
3119 if self.verbose:
3120 print "branch is %s" % branch
3122 self.updatedBranches.add(branch)
3124 if branch not in self.createdBranches:
3125 self.createdBranches.add(branch)
3126 parent = self.knownBranches[branch]
3127 if parent == branch:
3128 parent = ""
3129 else:
3130 fullBranch = self.projectName + branch
3131 if fullBranch not in self.p4BranchesInGit:
3132 if not self.silent:
3133 print("\n Importing new branch %s" % fullBranch);
3134 if self.importNewBranch(branch, change - 1):
3135 parent = ""
3136 self.p4BranchesInGit.append(fullBranch)
3137 if not self.silent:
3138 print("\n Resuming with change %s" % change);
3140 if self.verbose:
3141 print "parent determined through known branches: %s" % parent
3143 branch = self.gitRefForBranch(branch)
3144 parent = self.gitRefForBranch(parent)
3146 if self.verbose:
3147 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3149 if len(parent) == 0 and branch in self.initialParents:
3150 parent = self.initialParents[branch]
3151 del self.initialParents[branch]
3153 blob = None
3154 if len(parent) > 0:
3155 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3156 if self.verbose:
3157 print "Creating temporary branch: " + tempBranch
3158 self.commit(description, filesForCommit, tempBranch)
3159 self.tempBranches.append(tempBranch)
3160 self.checkpoint()
3161 blob = self.searchParent(parent, branch, tempBranch)
3162 if blob:
3163 self.commit(description, filesForCommit, branch, blob)
3164 else:
3165 if self.verbose:
3166 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3167 self.commit(description, filesForCommit, branch, parent)
3168 else:
3169 files = self.extractFilesFromCommit(description)
3170 self.commit(description, files, self.branch,
3171 self.initialParent)
3172 # only needed once, to connect to the previous commit
3173 self.initialParent = ""
3174 except IOError:
3175 print self.gitError.read()
3176 sys.exit(1)
3178 def importHeadRevision(self, revision):
3179 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3181 details = {}
3182 details["user"] = "git perforce import user"
3183 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3184 % (' '.join(self.depotPaths), revision))
3185 details["change"] = revision
3186 newestRevision = 0
3188 fileCnt = 0
3189 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3191 for info in p4CmdList(["files"] + fileArgs):
3193 if 'code' in info and info['code'] == 'error':
3194 sys.stderr.write("p4 returned an error: %s\n"
3195 % info['data'])
3196 if info['data'].find("must refer to client") >= 0:
3197 sys.stderr.write("This particular p4 error is misleading.\n")
3198 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3199 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3200 sys.exit(1)
3201 if 'p4ExitCode' in info:
3202 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3203 sys.exit(1)
3206 change = int(info["change"])
3207 if change > newestRevision:
3208 newestRevision = change
3210 if info["action"] in self.delete_actions:
3211 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3212 #fileCnt = fileCnt + 1
3213 continue
3215 for prop in ["depotFile", "rev", "action", "type" ]:
3216 details["%s%s" % (prop, fileCnt)] = info[prop]
3218 fileCnt = fileCnt + 1
3220 details["change"] = newestRevision
3222 # Use time from top-most change so that all git p4 clones of
3223 # the same p4 repo have the same commit SHA1s.
3224 res = p4_describe(newestRevision)
3225 details["time"] = res["time"]
3227 self.updateOptionDict(details)
3228 try:
3229 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3230 except IOError:
3231 print "IO error with git fast-import. Is your git version recent enough?"
3232 print self.gitError.read()
3235 def run(self, args):
3236 self.depotPaths = []
3237 self.changeRange = ""
3238 self.previousDepotPaths = []
3239 self.hasOrigin = False
3241 # map from branch depot path to parent branch
3242 self.knownBranches = {}
3243 self.initialParents = {}
3245 if self.importIntoRemotes:
3246 self.refPrefix = "refs/remotes/p4/"
3247 else:
3248 self.refPrefix = "refs/heads/p4/"
3250 if self.syncWithOrigin:
3251 self.hasOrigin = originP4BranchesExist()
3252 if self.hasOrigin:
3253 if not self.silent:
3254 print 'Syncing with origin first, using "git fetch origin"'
3255 system("git fetch origin")
3257 branch_arg_given = bool(self.branch)
3258 if len(self.branch) == 0:
3259 self.branch = self.refPrefix + "master"
3260 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3261 system("git update-ref %s refs/heads/p4" % self.branch)
3262 system("git branch -D p4")
3264 # accept either the command-line option, or the configuration variable
3265 if self.useClientSpec:
3266 # will use this after clone to set the variable
3267 self.useClientSpec_from_options = True
3268 else:
3269 if gitConfigBool("git-p4.useclientspec"):
3270 self.useClientSpec = True
3271 if self.useClientSpec:
3272 self.clientSpecDirs = getClientSpec()
3274 # TODO: should always look at previous commits,
3275 # merge with previous imports, if possible.
3276 if args == []:
3277 if self.hasOrigin:
3278 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3280 # branches holds mapping from branch name to sha1
3281 branches = p4BranchesInGit(self.importIntoRemotes)
3283 # restrict to just this one, disabling detect-branches
3284 if branch_arg_given:
3285 short = self.branch.split("/")[-1]
3286 if short in branches:
3287 self.p4BranchesInGit = [ short ]
3288 else:
3289 self.p4BranchesInGit = branches.keys()
3291 if len(self.p4BranchesInGit) > 1:
3292 if not self.silent:
3293 print "Importing from/into multiple branches"
3294 self.detectBranches = True
3295 for branch in branches.keys():
3296 self.initialParents[self.refPrefix + branch] = \
3297 branches[branch]
3299 if self.verbose:
3300 print "branches: %s" % self.p4BranchesInGit
3302 p4Change = 0
3303 for branch in self.p4BranchesInGit:
3304 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3306 settings = extractSettingsGitLog(logMsg)
3308 self.readOptions(settings)
3309 if (settings.has_key('depot-paths')
3310 and settings.has_key ('change')):
3311 change = int(settings['change']) + 1
3312 p4Change = max(p4Change, change)
3314 depotPaths = sorted(settings['depot-paths'])
3315 if self.previousDepotPaths == []:
3316 self.previousDepotPaths = depotPaths
3317 else:
3318 paths = []
3319 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3320 prev_list = prev.split("/")
3321 cur_list = cur.split("/")
3322 for i in range(0, min(len(cur_list), len(prev_list))):
3323 if cur_list[i] <> prev_list[i]:
3324 i = i - 1
3325 break
3327 paths.append ("/".join(cur_list[:i + 1]))
3329 self.previousDepotPaths = paths
3331 if p4Change > 0:
3332 self.depotPaths = sorted(self.previousDepotPaths)
3333 self.changeRange = "@%s,#head" % p4Change
3334 if not self.silent and not self.detectBranches:
3335 print "Performing incremental import into %s git branch" % self.branch
3337 # accept multiple ref name abbreviations:
3338 # refs/foo/bar/branch -> use it exactly
3339 # p4/branch -> prepend refs/remotes/ or refs/heads/
3340 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3341 if not self.branch.startswith("refs/"):
3342 if self.importIntoRemotes:
3343 prepend = "refs/remotes/"
3344 else:
3345 prepend = "refs/heads/"
3346 if not self.branch.startswith("p4/"):
3347 prepend += "p4/"
3348 self.branch = prepend + self.branch
3350 if len(args) == 0 and self.depotPaths:
3351 if not self.silent:
3352 print "Depot paths: %s" % ' '.join(self.depotPaths)
3353 else:
3354 if self.depotPaths and self.depotPaths != args:
3355 print ("previous import used depot path %s and now %s was specified. "
3356 "This doesn't work!" % (' '.join (self.depotPaths),
3357 ' '.join (args)))
3358 sys.exit(1)
3360 self.depotPaths = sorted(args)
3362 revision = ""
3363 self.users = {}
3365 # Make sure no revision specifiers are used when --changesfile
3366 # is specified.
3367 bad_changesfile = False
3368 if len(self.changesFile) > 0:
3369 for p in self.depotPaths:
3370 if p.find("@") >= 0 or p.find("#") >= 0:
3371 bad_changesfile = True
3372 break
3373 if bad_changesfile:
3374 die("Option --changesfile is incompatible with revision specifiers")
3376 newPaths = []
3377 for p in self.depotPaths:
3378 if p.find("@") != -1:
3379 atIdx = p.index("@")
3380 self.changeRange = p[atIdx:]
3381 if self.changeRange == "@all":
3382 self.changeRange = ""
3383 elif ',' not in self.changeRange:
3384 revision = self.changeRange
3385 self.changeRange = ""
3386 p = p[:atIdx]
3387 elif p.find("#") != -1:
3388 hashIdx = p.index("#")
3389 revision = p[hashIdx:]
3390 p = p[:hashIdx]
3391 elif self.previousDepotPaths == []:
3392 # pay attention to changesfile, if given, else import
3393 # the entire p4 tree at the head revision
3394 if len(self.changesFile) == 0:
3395 revision = "#head"
3397 p = re.sub ("\.\.\.$", "", p)
3398 if not p.endswith("/"):
3399 p += "/"
3401 newPaths.append(p)
3403 self.depotPaths = newPaths
3405 # --detect-branches may change this for each branch
3406 self.branchPrefixes = self.depotPaths
3408 self.loadUserMapFromCache()
3409 self.labels = {}
3410 if self.detectLabels:
3411 self.getLabels();
3413 if self.detectBranches:
3414 ## FIXME - what's a P4 projectName ?
3415 self.projectName = self.guessProjectName()
3417 if self.hasOrigin:
3418 self.getBranchMappingFromGitBranches()
3419 else:
3420 self.getBranchMapping()
3421 if self.verbose:
3422 print "p4-git branches: %s" % self.p4BranchesInGit
3423 print "initial parents: %s" % self.initialParents
3424 for b in self.p4BranchesInGit:
3425 if b != "master":
3427 ## FIXME
3428 b = b[len(self.projectName):]
3429 self.createdBranches.add(b)
3431 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3433 self.importProcess = subprocess.Popen(["git", "fast-import"],
3434 stdin=subprocess.PIPE,
3435 stdout=subprocess.PIPE,
3436 stderr=subprocess.PIPE);
3437 self.gitOutput = self.importProcess.stdout
3438 self.gitStream = self.importProcess.stdin
3439 self.gitError = self.importProcess.stderr
3441 if revision:
3442 self.importHeadRevision(revision)
3443 else:
3444 changes = []
3446 if len(self.changesFile) > 0:
3447 output = open(self.changesFile).readlines()
3448 changeSet = set()
3449 for line in output:
3450 changeSet.add(int(line))
3452 for change in changeSet:
3453 changes.append(change)
3455 changes.sort()
3456 else:
3457 # catch "git p4 sync" with no new branches, in a repo that
3458 # does not have any existing p4 branches
3459 if len(args) == 0:
3460 if not self.p4BranchesInGit:
3461 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3463 # The default branch is master, unless --branch is used to
3464 # specify something else. Make sure it exists, or complain
3465 # nicely about how to use --branch.
3466 if not self.detectBranches:
3467 if not branch_exists(self.branch):
3468 if branch_arg_given:
3469 die("Error: branch %s does not exist." % self.branch)
3470 else:
3471 die("Error: no branch %s; perhaps specify one with --branch." %
3472 self.branch)
3474 if self.verbose:
3475 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3476 self.changeRange)
3477 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3479 if len(self.maxChanges) > 0:
3480 changes = changes[:min(int(self.maxChanges), len(changes))]
3482 if len(changes) == 0:
3483 if not self.silent:
3484 print "No changes to import!"
3485 else:
3486 if not self.silent and not self.detectBranches:
3487 print "Import destination: %s" % self.branch
3489 self.updatedBranches = set()
3491 if not self.detectBranches:
3492 if args:
3493 # start a new branch
3494 self.initialParent = ""
3495 else:
3496 # build on a previous revision
3497 self.initialParent = parseRevision(self.branch)
3499 self.importChanges(changes)
3501 if not self.silent:
3502 print ""
3503 if len(self.updatedBranches) > 0:
3504 sys.stdout.write("Updated branches: ")
3505 for b in self.updatedBranches:
3506 sys.stdout.write("%s " % b)
3507 sys.stdout.write("\n")
3509 if gitConfigBool("git-p4.importLabels"):
3510 self.importLabels = True
3512 if self.importLabels:
3513 p4Labels = getP4Labels(self.depotPaths)
3514 gitTags = getGitTags()
3516 missingP4Labels = p4Labels - gitTags
3517 self.importP4Labels(self.gitStream, missingP4Labels)
3519 self.gitStream.close()
3520 if self.importProcess.wait() != 0:
3521 die("fast-import failed: %s" % self.gitError.read())
3522 self.gitOutput.close()
3523 self.gitError.close()
3525 # Cleanup temporary branches created during import
3526 if self.tempBranches != []:
3527 for branch in self.tempBranches:
3528 read_pipe("git update-ref -d %s" % branch)
3529 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3531 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3532 # a convenient shortcut refname "p4".
3533 if self.importIntoRemotes:
3534 head_ref = self.refPrefix + "HEAD"
3535 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3536 system(["git", "symbolic-ref", head_ref, self.branch])
3538 return True
3540 class P4Rebase(Command):
3541 def __init__(self):
3542 Command.__init__(self)
3543 self.options = [
3544 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3546 self.importLabels = False
3547 self.description = ("Fetches the latest revision from perforce and "
3548 + "rebases the current work (branch) against it")
3550 def run(self, args):
3551 sync = P4Sync()
3552 sync.importLabels = self.importLabels
3553 sync.run([])
3555 return self.rebase()
3557 def rebase(self):
3558 if os.system("git update-index --refresh") != 0:
3559 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3560 if len(read_pipe("git diff-index HEAD --")) > 0:
3561 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3563 [upstream, settings] = findUpstreamBranchPoint()
3564 if len(upstream) == 0:
3565 die("Cannot find upstream branchpoint for rebase")
3567 # the branchpoint may be p4/foo~3, so strip off the parent
3568 upstream = re.sub("~[0-9]+$", "", upstream)
3570 print "Rebasing the current branch onto %s" % upstream
3571 oldHead = read_pipe("git rev-parse HEAD").strip()
3572 system("git rebase %s" % upstream)
3573 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3574 return True
3576 class P4Clone(P4Sync):
3577 def __init__(self):
3578 P4Sync.__init__(self)
3579 self.description = "Creates a new git repository and imports from Perforce into it"
3580 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3581 self.options += [
3582 optparse.make_option("--destination", dest="cloneDestination",
3583 action='store', default=None,
3584 help="where to leave result of the clone"),
3585 optparse.make_option("--bare", dest="cloneBare",
3586 action="store_true", default=False),
3588 self.cloneDestination = None
3589 self.needsGit = False
3590 self.cloneBare = False
3592 def defaultDestination(self, args):
3593 ## TODO: use common prefix of args?
3594 depotPath = args[0]
3595 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3596 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3597 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3598 depotDir = re.sub(r"/$", "", depotDir)
3599 return os.path.split(depotDir)[1]
3601 def run(self, args):
3602 if len(args) < 1:
3603 return False
3605 if self.keepRepoPath and not self.cloneDestination:
3606 sys.stderr.write("Must specify destination for --keep-path\n")
3607 sys.exit(1)
3609 depotPaths = args
3611 if not self.cloneDestination and len(depotPaths) > 1:
3612 self.cloneDestination = depotPaths[-1]
3613 depotPaths = depotPaths[:-1]
3615 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3616 for p in depotPaths:
3617 if not p.startswith("//"):
3618 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3619 return False
3621 if not self.cloneDestination:
3622 self.cloneDestination = self.defaultDestination(args)
3624 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3626 if not os.path.exists(self.cloneDestination):
3627 os.makedirs(self.cloneDestination)
3628 chdir(self.cloneDestination)
3630 init_cmd = [ "git", "init" ]
3631 if self.cloneBare:
3632 init_cmd.append("--bare")
3633 retcode = subprocess.call(init_cmd)
3634 if retcode:
3635 raise CalledProcessError(retcode, init_cmd)
3637 if not P4Sync.run(self, depotPaths):
3638 return False
3640 # create a master branch and check out a work tree
3641 if gitBranchExists(self.branch):
3642 system([ "git", "branch", "master", self.branch ])
3643 if not self.cloneBare:
3644 system([ "git", "checkout", "-f" ])
3645 else:
3646 print 'Not checking out any branch, use ' \
3647 '"git checkout -q -b master <branch>"'
3649 # auto-set this variable if invoked with --use-client-spec
3650 if self.useClientSpec_from_options:
3651 system("git config --bool git-p4.useclientspec true")
3653 return True
3655 class P4Branches(Command):
3656 def __init__(self):
3657 Command.__init__(self)
3658 self.options = [ ]
3659 self.description = ("Shows the git branches that hold imports and their "
3660 + "corresponding perforce depot paths")
3661 self.verbose = False
3663 def run(self, args):
3664 if originP4BranchesExist():
3665 createOrUpdateBranchesFromOrigin()
3667 cmdline = "git rev-parse --symbolic "
3668 cmdline += " --remotes"
3670 for line in read_pipe_lines(cmdline):
3671 line = line.strip()
3673 if not line.startswith('p4/') or line == "p4/HEAD":
3674 continue
3675 branch = line
3677 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3678 settings = extractSettingsGitLog(log)
3680 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3681 return True
3683 class HelpFormatter(optparse.IndentedHelpFormatter):
3684 def __init__(self):
3685 optparse.IndentedHelpFormatter.__init__(self)
3687 def format_description(self, description):
3688 if description:
3689 return description + "\n"
3690 else:
3691 return ""
3693 def printUsage(commands):
3694 print "usage: %s <command> [options]" % sys.argv[0]
3695 print ""
3696 print "valid commands: %s" % ", ".join(commands)
3697 print ""
3698 print "Try %s <command> --help for command specific help." % sys.argv[0]
3699 print ""
3701 commands = {
3702 "debug" : P4Debug,
3703 "submit" : P4Submit,
3704 "commit" : P4Submit,
3705 "sync" : P4Sync,
3706 "rebase" : P4Rebase,
3707 "clone" : P4Clone,
3708 "rollback" : P4RollBack,
3709 "branches" : P4Branches
3713 def main():
3714 if len(sys.argv[1:]) == 0:
3715 printUsage(commands.keys())
3716 sys.exit(2)
3718 cmdName = sys.argv[1]
3719 try:
3720 klass = commands[cmdName]
3721 cmd = klass()
3722 except KeyError:
3723 print "unknown command %s" % cmdName
3724 print ""
3725 printUsage(commands.keys())
3726 sys.exit(2)
3728 options = cmd.options
3729 cmd.gitdir = os.environ.get("GIT_DIR", None)
3731 args = sys.argv[2:]
3733 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3734 if cmd.needsGit:
3735 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3737 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3738 options,
3739 description = cmd.description,
3740 formatter = HelpFormatter())
3742 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3743 global verbose
3744 verbose = cmd.verbose
3745 if cmd.needsGit:
3746 if cmd.gitdir == None:
3747 cmd.gitdir = os.path.abspath(".git")
3748 if not isValidGitDir(cmd.gitdir):
3749 # "rev-parse --git-dir" without arguments will try $PWD/.git
3750 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3751 if os.path.exists(cmd.gitdir):
3752 cdup = read_pipe("git rev-parse --show-cdup").strip()
3753 if len(cdup) > 0:
3754 chdir(cdup);
3756 if not isValidGitDir(cmd.gitdir):
3757 if isValidGitDir(cmd.gitdir + "/.git"):
3758 cmd.gitdir += "/.git"
3759 else:
3760 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3762 # so git commands invoked from the P4 workspace will succeed
3763 os.environ["GIT_DIR"] = cmd.gitdir
3765 if not cmd.run(args):
3766 parser.print_help()
3767 sys.exit(2)
3770 if __name__ == '__main__':
3771 main()