Merge branch 'ag/rebase-p'
[alt-git.git] / git-p4.py
blob0354d4df5cadd5145245d755a8868608b849719f
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
30 try:
31 from subprocess import CalledProcessError
32 except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
45 verbose = False
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
50 # The block size is reduced automatically if required
51 defaultBlockSize = 1<<20
53 p4_access_checked = False
55 def p4_build_cmd(cmd):
56 """Build a suitable p4 command line.
58 This consolidates building and returning a p4 command line into one
59 location. It means that hooking into the environment, or other configuration
60 can be done more easily.
61 """
62 real_cmd = ["p4"]
64 user = gitConfig("git-p4.user")
65 if len(user) > 0:
66 real_cmd += ["-u",user]
68 password = gitConfig("git-p4.password")
69 if len(password) > 0:
70 real_cmd += ["-P", password]
72 port = gitConfig("git-p4.port")
73 if len(port) > 0:
74 real_cmd += ["-p", port]
76 host = gitConfig("git-p4.host")
77 if len(host) > 0:
78 real_cmd += ["-H", host]
80 client = gitConfig("git-p4.client")
81 if len(client) > 0:
82 real_cmd += ["-c", client]
84 retries = gitConfigInt("git-p4.retries")
85 if retries is None:
86 # Perform 3 retries by default
87 retries = 3
88 if retries > 0:
89 # Provide a way to not pass this option by setting git-p4.retries to 0
90 real_cmd += ["-r", str(retries)]
92 if isinstance(cmd,basestring):
93 real_cmd = ' '.join(real_cmd) + ' ' + cmd
94 else:
95 real_cmd += cmd
97 # now check that we can actually talk to the server
98 global p4_access_checked
99 if not p4_access_checked:
100 p4_access_checked = True # suppress access checks in p4_check_access itself
101 p4_check_access()
103 return real_cmd
105 def git_dir(path):
106 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
107 This won't automatically add ".git" to a directory.
109 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
110 if not d or len(d) == 0:
111 return None
112 else:
113 return d
115 def chdir(path, is_client_path=False):
116 """Do chdir to the given path, and set the PWD environment
117 variable for use by P4. It does not look at getcwd() output.
118 Since we're not using the shell, it is necessary to set the
119 PWD environment variable explicitly.
121 Normally, expand the path to force it to be absolute. This
122 addresses the use of relative path names inside P4 settings,
123 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
124 as given; it looks for .p4config using PWD.
126 If is_client_path, the path was handed to us directly by p4,
127 and may be a symbolic link. Do not call os.getcwd() in this
128 case, because it will cause p4 to think that PWD is not inside
129 the client path.
132 os.chdir(path)
133 if not is_client_path:
134 path = os.getcwd()
135 os.environ['PWD'] = path
137 def calcDiskFree():
138 """Return free space in bytes on the disk of the given dirname."""
139 if platform.system() == 'Windows':
140 free_bytes = ctypes.c_ulonglong(0)
141 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
142 return free_bytes.value
143 else:
144 st = os.statvfs(os.getcwd())
145 return st.f_bavail * st.f_frsize
147 def die(msg):
148 if verbose:
149 raise Exception(msg)
150 else:
151 sys.stderr.write(msg + "\n")
152 sys.exit(1)
154 def write_pipe(c, stdin):
155 if verbose:
156 sys.stderr.write('Writing pipe: %s\n' % str(c))
158 expand = isinstance(c,basestring)
159 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
160 pipe = p.stdin
161 val = pipe.write(stdin)
162 pipe.close()
163 if p.wait():
164 die('Command failed: %s' % str(c))
166 return val
168 def p4_write_pipe(c, stdin):
169 real_cmd = p4_build_cmd(c)
170 return write_pipe(real_cmd, stdin)
172 def read_pipe_full(c):
173 """ Read output from command. Returns a tuple
174 of the return status, stdout text and stderr
175 text.
177 if verbose:
178 sys.stderr.write('Reading pipe: %s\n' % str(c))
180 expand = isinstance(c,basestring)
181 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
182 (out, err) = p.communicate()
183 return (p.returncode, out, err)
185 def read_pipe(c, ignore_error=False):
186 """ Read output from command. Returns the output text on
187 success. On failure, terminates execution, unless
188 ignore_error is True, when it returns an empty string.
190 (retcode, out, err) = read_pipe_full(c)
191 if retcode != 0:
192 if ignore_error:
193 out = ""
194 else:
195 die('Command failed: %s\nError: %s' % (str(c), err))
196 return out
198 def read_pipe_text(c):
199 """ Read output from a command with trailing whitespace stripped.
200 On error, returns None.
202 (retcode, out, err) = read_pipe_full(c)
203 if retcode != 0:
204 return None
205 else:
206 return out.rstrip()
208 def p4_read_pipe(c, ignore_error=False):
209 real_cmd = p4_build_cmd(c)
210 return read_pipe(real_cmd, ignore_error)
212 def read_pipe_lines(c):
213 if verbose:
214 sys.stderr.write('Reading pipe: %s\n' % str(c))
216 expand = isinstance(c, basestring)
217 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
218 pipe = p.stdout
219 val = pipe.readlines()
220 if pipe.close() or p.wait():
221 die('Command failed: %s' % str(c))
223 return val
225 def p4_read_pipe_lines(c):
226 """Specifically invoke p4 on the command supplied. """
227 real_cmd = p4_build_cmd(c)
228 return read_pipe_lines(real_cmd)
230 def p4_has_command(cmd):
231 """Ask p4 for help on this command. If it returns an error, the
232 command does not exist in this version of p4."""
233 real_cmd = p4_build_cmd(["help", cmd])
234 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
235 stderr=subprocess.PIPE)
236 p.communicate()
237 return p.returncode == 0
239 def p4_has_move_command():
240 """See if the move command exists, that it supports -k, and that
241 it has not been administratively disabled. The arguments
242 must be correct, but the filenames do not have to exist. Use
243 ones with wildcards so even if they exist, it will fail."""
245 if not p4_has_command("move"):
246 return False
247 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
248 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
249 (out, err) = p.communicate()
250 # return code will be 1 in either case
251 if err.find("Invalid option") >= 0:
252 return False
253 if err.find("disabled") >= 0:
254 return False
255 # assume it failed because @... was invalid changelist
256 return True
258 def system(cmd, ignore_error=False):
259 expand = isinstance(cmd,basestring)
260 if verbose:
261 sys.stderr.write("executing %s\n" % str(cmd))
262 retcode = subprocess.call(cmd, shell=expand)
263 if retcode and not ignore_error:
264 raise CalledProcessError(retcode, cmd)
266 return retcode
268 def p4_system(cmd):
269 """Specifically invoke p4 as the system command. """
270 real_cmd = p4_build_cmd(cmd)
271 expand = isinstance(real_cmd, basestring)
272 retcode = subprocess.call(real_cmd, shell=expand)
273 if retcode:
274 raise CalledProcessError(retcode, real_cmd)
276 def die_bad_access(s):
277 die("failure accessing depot: {0}".format(s.rstrip()))
279 def p4_check_access(min_expiration=1):
280 """ Check if we can access Perforce - account still logged in
282 results = p4CmdList(["login", "-s"])
284 if len(results) == 0:
285 # should never get here: always get either some results, or a p4ExitCode
286 assert("could not parse response from perforce")
288 result = results[0]
290 if 'p4ExitCode' in result:
291 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
292 die_bad_access("could not run p4")
294 code = result.get("code")
295 if not code:
296 # we get here if we couldn't connect and there was nothing to unmarshal
297 die_bad_access("could not connect")
299 elif code == "stat":
300 expiry = result.get("TicketExpiration")
301 if expiry:
302 expiry = int(expiry)
303 if expiry > min_expiration:
304 # ok to carry on
305 return
306 else:
307 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
309 else:
310 # account without a timeout - all ok
311 return
313 elif code == "error":
314 data = result.get("data")
315 if data:
316 die_bad_access("p4 error: {0}".format(data))
317 else:
318 die_bad_access("unknown error")
319 else:
320 die_bad_access("unknown error code {0}".format(code))
322 _p4_version_string = None
323 def p4_version_string():
324 """Read the version string, showing just the last line, which
325 hopefully is the interesting version bit.
327 $ p4 -V
328 Perforce - The Fast Software Configuration Management System.
329 Copyright 1995-2011 Perforce Software. All rights reserved.
330 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
332 global _p4_version_string
333 if not _p4_version_string:
334 a = p4_read_pipe_lines(["-V"])
335 _p4_version_string = a[-1].rstrip()
336 return _p4_version_string
338 def p4_integrate(src, dest):
339 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
341 def p4_sync(f, *options):
342 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
344 def p4_add(f):
345 # forcibly add file names with wildcards
346 if wildcard_present(f):
347 p4_system(["add", "-f", f])
348 else:
349 p4_system(["add", f])
351 def p4_delete(f):
352 p4_system(["delete", wildcard_encode(f)])
354 def p4_edit(f, *options):
355 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
357 def p4_revert(f):
358 p4_system(["revert", wildcard_encode(f)])
360 def p4_reopen(type, f):
361 p4_system(["reopen", "-t", type, wildcard_encode(f)])
363 def p4_reopen_in_change(changelist, files):
364 cmd = ["reopen", "-c", str(changelist)] + files
365 p4_system(cmd)
367 def p4_move(src, dest):
368 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
370 def p4_last_change():
371 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
372 return int(results[0]['change'])
374 def p4_describe(change, shelved=False):
375 """Make sure it returns a valid result by checking for
376 the presence of field "time". Return a dict of the
377 results."""
379 cmd = ["describe", "-s"]
380 if shelved:
381 cmd += ["-S"]
382 cmd += [str(change)]
384 ds = p4CmdList(cmd, skip_info=True)
385 if len(ds) != 1:
386 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
388 d = ds[0]
390 if "p4ExitCode" in d:
391 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
392 str(d)))
393 if "code" in d:
394 if d["code"] == "error":
395 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
397 if "time" not in d:
398 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
400 return d
403 # Canonicalize the p4 type and return a tuple of the
404 # base type, plus any modifiers. See "p4 help filetypes"
405 # for a list and explanation.
407 def split_p4_type(p4type):
409 p4_filetypes_historical = {
410 "ctempobj": "binary+Sw",
411 "ctext": "text+C",
412 "cxtext": "text+Cx",
413 "ktext": "text+k",
414 "kxtext": "text+kx",
415 "ltext": "text+F",
416 "tempobj": "binary+FSw",
417 "ubinary": "binary+F",
418 "uresource": "resource+F",
419 "uxbinary": "binary+Fx",
420 "xbinary": "binary+x",
421 "xltext": "text+Fx",
422 "xtempobj": "binary+Swx",
423 "xtext": "text+x",
424 "xunicode": "unicode+x",
425 "xutf16": "utf16+x",
427 if p4type in p4_filetypes_historical:
428 p4type = p4_filetypes_historical[p4type]
429 mods = ""
430 s = p4type.split("+")
431 base = s[0]
432 mods = ""
433 if len(s) > 1:
434 mods = s[1]
435 return (base, mods)
438 # return the raw p4 type of a file (text, text+ko, etc)
440 def p4_type(f):
441 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
442 return results[0]['headType']
445 # Given a type base and modifier, return a regexp matching
446 # the keywords that can be expanded in the file
448 def p4_keywords_regexp_for_type(base, type_mods):
449 if base in ("text", "unicode", "binary"):
450 kwords = None
451 if "ko" in type_mods:
452 kwords = 'Id|Header'
453 elif "k" in type_mods:
454 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
455 else:
456 return None
457 pattern = r"""
458 \$ # Starts with a dollar, followed by...
459 (%s) # one of the keywords, followed by...
460 (:[^$\n]+)? # possibly an old expansion, followed by...
461 \$ # another dollar
462 """ % kwords
463 return pattern
464 else:
465 return None
468 # Given a file, return a regexp matching the possible
469 # RCS keywords that will be expanded, or None for files
470 # with kw expansion turned off.
472 def p4_keywords_regexp_for_file(file):
473 if not os.path.exists(file):
474 return None
475 else:
476 (type_base, type_mods) = split_p4_type(p4_type(file))
477 return p4_keywords_regexp_for_type(type_base, type_mods)
479 def setP4ExecBit(file, mode):
480 # Reopens an already open file and changes the execute bit to match
481 # the execute bit setting in the passed in mode.
483 p4Type = "+x"
485 if not isModeExec(mode):
486 p4Type = getP4OpenedType(file)
487 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
488 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
489 if p4Type[-1] == "+":
490 p4Type = p4Type[0:-1]
492 p4_reopen(p4Type, file)
494 def getP4OpenedType(file):
495 # Returns the perforce file type for the given file.
497 result = p4_read_pipe(["opened", wildcard_encode(file)])
498 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
499 if match:
500 return match.group(1)
501 else:
502 die("Could not determine file type for %s (result: '%s')" % (file, result))
504 # Return the set of all p4 labels
505 def getP4Labels(depotPaths):
506 labels = set()
507 if isinstance(depotPaths,basestring):
508 depotPaths = [depotPaths]
510 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
511 label = l['label']
512 labels.add(label)
514 return labels
516 # Return the set of all git tags
517 def getGitTags():
518 gitTags = set()
519 for line in read_pipe_lines(["git", "tag"]):
520 tag = line.strip()
521 gitTags.add(tag)
522 return gitTags
524 def diffTreePattern():
525 # This is a simple generator for the diff tree regex pattern. This could be
526 # a class variable if this and parseDiffTreeEntry were a part of a class.
527 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
528 while True:
529 yield pattern
531 def parseDiffTreeEntry(entry):
532 """Parses a single diff tree entry into its component elements.
534 See git-diff-tree(1) manpage for details about the format of the diff
535 output. This method returns a dictionary with the following elements:
537 src_mode - The mode of the source file
538 dst_mode - The mode of the destination file
539 src_sha1 - The sha1 for the source file
540 dst_sha1 - The sha1 fr the destination file
541 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
542 status_score - The score for the status (applicable for 'C' and 'R'
543 statuses). This is None if there is no score.
544 src - The path for the source file.
545 dst - The path for the destination file. This is only present for
546 copy or renames. If it is not present, this is None.
548 If the pattern is not matched, None is returned."""
550 match = diffTreePattern().next().match(entry)
551 if match:
552 return {
553 'src_mode': match.group(1),
554 'dst_mode': match.group(2),
555 'src_sha1': match.group(3),
556 'dst_sha1': match.group(4),
557 'status': match.group(5),
558 'status_score': match.group(6),
559 'src': match.group(7),
560 'dst': match.group(10)
562 return None
564 def isModeExec(mode):
565 # Returns True if the given git mode represents an executable file,
566 # otherwise False.
567 return mode[-3:] == "755"
569 class P4Exception(Exception):
570 """ Base class for exceptions from the p4 client """
571 def __init__(self, exit_code):
572 self.p4ExitCode = exit_code
574 class P4ServerException(P4Exception):
575 """ Base class for exceptions where we get some kind of marshalled up result from the server """
576 def __init__(self, exit_code, p4_result):
577 super(P4ServerException, self).__init__(exit_code)
578 self.p4_result = p4_result
579 self.code = p4_result[0]['code']
580 self.data = p4_result[0]['data']
582 class P4RequestSizeException(P4ServerException):
583 """ One of the maxresults or maxscanrows errors """
584 def __init__(self, exit_code, p4_result, limit):
585 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
586 self.limit = limit
588 def isModeExecChanged(src_mode, dst_mode):
589 return isModeExec(src_mode) != isModeExec(dst_mode)
591 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
592 errors_as_exceptions=False):
594 if isinstance(cmd,basestring):
595 cmd = "-G " + cmd
596 expand = True
597 else:
598 cmd = ["-G"] + cmd
599 expand = False
601 cmd = p4_build_cmd(cmd)
602 if verbose:
603 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
605 # Use a temporary file to avoid deadlocks without
606 # subprocess.communicate(), which would put another copy
607 # of stdout into memory.
608 stdin_file = None
609 if stdin is not None:
610 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
611 if isinstance(stdin,basestring):
612 stdin_file.write(stdin)
613 else:
614 for i in stdin:
615 stdin_file.write(i + '\n')
616 stdin_file.flush()
617 stdin_file.seek(0)
619 p4 = subprocess.Popen(cmd,
620 shell=expand,
621 stdin=stdin_file,
622 stdout=subprocess.PIPE)
624 result = []
625 try:
626 while True:
627 entry = marshal.load(p4.stdout)
628 if skip_info:
629 if 'code' in entry and entry['code'] == 'info':
630 continue
631 if cb is not None:
632 cb(entry)
633 else:
634 result.append(entry)
635 except EOFError:
636 pass
637 exitCode = p4.wait()
638 if exitCode != 0:
639 if errors_as_exceptions:
640 if len(result) > 0:
641 data = result[0].get('data')
642 if data:
643 m = re.search('Too many rows scanned \(over (\d+)\)', data)
644 if not m:
645 m = re.search('Request too large \(over (\d+)\)', data)
647 if m:
648 limit = int(m.group(1))
649 raise P4RequestSizeException(exitCode, result, limit)
651 raise P4ServerException(exitCode, result)
652 else:
653 raise P4Exception(exitCode)
654 else:
655 entry = {}
656 entry["p4ExitCode"] = exitCode
657 result.append(entry)
659 return result
661 def p4Cmd(cmd):
662 list = p4CmdList(cmd)
663 result = {}
664 for entry in list:
665 result.update(entry)
666 return result;
668 def p4Where(depotPath):
669 if not depotPath.endswith("/"):
670 depotPath += "/"
671 depotPathLong = depotPath + "..."
672 outputList = p4CmdList(["where", depotPathLong])
673 output = None
674 for entry in outputList:
675 if "depotFile" in entry:
676 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
677 # The base path always ends with "/...".
678 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
679 output = entry
680 break
681 elif "data" in entry:
682 data = entry.get("data")
683 space = data.find(" ")
684 if data[:space] == depotPath:
685 output = entry
686 break
687 if output == None:
688 return ""
689 if output["code"] == "error":
690 return ""
691 clientPath = ""
692 if "path" in output:
693 clientPath = output.get("path")
694 elif "data" in output:
695 data = output.get("data")
696 lastSpace = data.rfind(" ")
697 clientPath = data[lastSpace + 1:]
699 if clientPath.endswith("..."):
700 clientPath = clientPath[:-3]
701 return clientPath
703 def currentGitBranch():
704 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
706 def isValidGitDir(path):
707 return git_dir(path) != None
709 def parseRevision(ref):
710 return read_pipe("git rev-parse %s" % ref).strip()
712 def branchExists(ref):
713 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
714 ignore_error=True)
715 return len(rev) > 0
717 def extractLogMessageFromGitCommit(commit):
718 logMessage = ""
720 ## fixme: title is first line of commit, not 1st paragraph.
721 foundTitle = False
722 for log in read_pipe_lines("git cat-file commit %s" % commit):
723 if not foundTitle:
724 if len(log) == 1:
725 foundTitle = True
726 continue
728 logMessage += log
729 return logMessage
731 def extractSettingsGitLog(log):
732 values = {}
733 for line in log.split("\n"):
734 line = line.strip()
735 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
736 if not m:
737 continue
739 assignments = m.group(1).split (':')
740 for a in assignments:
741 vals = a.split ('=')
742 key = vals[0].strip()
743 val = ('='.join (vals[1:])).strip()
744 if val.endswith ('\"') and val.startswith('"'):
745 val = val[1:-1]
747 values[key] = val
749 paths = values.get("depot-paths")
750 if not paths:
751 paths = values.get("depot-path")
752 if paths:
753 values['depot-paths'] = paths.split(',')
754 return values
756 def gitBranchExists(branch):
757 proc = subprocess.Popen(["git", "rev-parse", branch],
758 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
759 return proc.wait() == 0;
761 def gitUpdateRef(ref, newvalue):
762 subprocess.check_call(["git", "update-ref", ref, newvalue])
764 def gitDeleteRef(ref):
765 subprocess.check_call(["git", "update-ref", "-d", ref])
767 _gitConfig = {}
769 def gitConfig(key, typeSpecifier=None):
770 if not _gitConfig.has_key(key):
771 cmd = [ "git", "config" ]
772 if typeSpecifier:
773 cmd += [ typeSpecifier ]
774 cmd += [ key ]
775 s = read_pipe(cmd, ignore_error=True)
776 _gitConfig[key] = s.strip()
777 return _gitConfig[key]
779 def gitConfigBool(key):
780 """Return a bool, using git config --bool. It is True only if the
781 variable is set to true, and False if set to false or not present
782 in the config."""
784 if not _gitConfig.has_key(key):
785 _gitConfig[key] = gitConfig(key, '--bool') == "true"
786 return _gitConfig[key]
788 def gitConfigInt(key):
789 if not _gitConfig.has_key(key):
790 cmd = [ "git", "config", "--int", key ]
791 s = read_pipe(cmd, ignore_error=True)
792 v = s.strip()
793 try:
794 _gitConfig[key] = int(gitConfig(key, '--int'))
795 except ValueError:
796 _gitConfig[key] = None
797 return _gitConfig[key]
799 def gitConfigList(key):
800 if not _gitConfig.has_key(key):
801 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
802 _gitConfig[key] = s.strip().splitlines()
803 if _gitConfig[key] == ['']:
804 _gitConfig[key] = []
805 return _gitConfig[key]
807 def p4BranchesInGit(branchesAreInRemotes=True):
808 """Find all the branches whose names start with "p4/", looking
809 in remotes or heads as specified by the argument. Return
810 a dictionary of { branch: revision } for each one found.
811 The branch names are the short names, without any
812 "p4/" prefix."""
814 branches = {}
816 cmdline = "git rev-parse --symbolic "
817 if branchesAreInRemotes:
818 cmdline += "--remotes"
819 else:
820 cmdline += "--branches"
822 for line in read_pipe_lines(cmdline):
823 line = line.strip()
825 # only import to p4/
826 if not line.startswith('p4/'):
827 continue
828 # special symbolic ref to p4/master
829 if line == "p4/HEAD":
830 continue
832 # strip off p4/ prefix
833 branch = line[len("p4/"):]
835 branches[branch] = parseRevision(line)
837 return branches
839 def branch_exists(branch):
840 """Make sure that the given ref name really exists."""
842 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
843 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
844 out, _ = p.communicate()
845 if p.returncode:
846 return False
847 # expect exactly one line of output: the branch name
848 return out.rstrip() == branch
850 def findUpstreamBranchPoint(head = "HEAD"):
851 branches = p4BranchesInGit()
852 # map from depot-path to branch name
853 branchByDepotPath = {}
854 for branch in branches.keys():
855 tip = branches[branch]
856 log = extractLogMessageFromGitCommit(tip)
857 settings = extractSettingsGitLog(log)
858 if settings.has_key("depot-paths"):
859 paths = ",".join(settings["depot-paths"])
860 branchByDepotPath[paths] = "remotes/p4/" + branch
862 settings = None
863 parent = 0
864 while parent < 65535:
865 commit = head + "~%s" % parent
866 log = extractLogMessageFromGitCommit(commit)
867 settings = extractSettingsGitLog(log)
868 if settings.has_key("depot-paths"):
869 paths = ",".join(settings["depot-paths"])
870 if branchByDepotPath.has_key(paths):
871 return [branchByDepotPath[paths], settings]
873 parent = parent + 1
875 return ["", settings]
877 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
878 if not silent:
879 print ("Creating/updating branch(es) in %s based on origin branch(es)"
880 % localRefPrefix)
882 originPrefix = "origin/p4/"
884 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
885 line = line.strip()
886 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
887 continue
889 headName = line[len(originPrefix):]
890 remoteHead = localRefPrefix + headName
891 originHead = line
893 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
894 if (not original.has_key('depot-paths')
895 or not original.has_key('change')):
896 continue
898 update = False
899 if not gitBranchExists(remoteHead):
900 if verbose:
901 print "creating %s" % remoteHead
902 update = True
903 else:
904 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
905 if settings.has_key('change') > 0:
906 if settings['depot-paths'] == original['depot-paths']:
907 originP4Change = int(original['change'])
908 p4Change = int(settings['change'])
909 if originP4Change > p4Change:
910 print ("%s (%s) is newer than %s (%s). "
911 "Updating p4 branch from origin."
912 % (originHead, originP4Change,
913 remoteHead, p4Change))
914 update = True
915 else:
916 print ("Ignoring: %s was imported from %s while "
917 "%s was imported from %s"
918 % (originHead, ','.join(original['depot-paths']),
919 remoteHead, ','.join(settings['depot-paths'])))
921 if update:
922 system("git update-ref %s %s" % (remoteHead, originHead))
924 def originP4BranchesExist():
925 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
928 def p4ParseNumericChangeRange(parts):
929 changeStart = int(parts[0][1:])
930 if parts[1] == '#head':
931 changeEnd = p4_last_change()
932 else:
933 changeEnd = int(parts[1])
935 return (changeStart, changeEnd)
937 def chooseBlockSize(blockSize):
938 if blockSize:
939 return blockSize
940 else:
941 return defaultBlockSize
943 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
944 assert depotPaths
946 # Parse the change range into start and end. Try to find integer
947 # revision ranges as these can be broken up into blocks to avoid
948 # hitting server-side limits (maxrows, maxscanresults). But if
949 # that doesn't work, fall back to using the raw revision specifier
950 # strings, without using block mode.
952 if changeRange is None or changeRange == '':
953 changeStart = 1
954 changeEnd = p4_last_change()
955 block_size = chooseBlockSize(requestedBlockSize)
956 else:
957 parts = changeRange.split(',')
958 assert len(parts) == 2
959 try:
960 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
961 block_size = chooseBlockSize(requestedBlockSize)
962 except ValueError:
963 changeStart = parts[0][1:]
964 changeEnd = parts[1]
965 if requestedBlockSize:
966 die("cannot use --changes-block-size with non-numeric revisions")
967 block_size = None
969 changes = set()
971 # Retrieve changes a block at a time, to prevent running
972 # into a MaxResults/MaxScanRows error from the server. If
973 # we _do_ hit one of those errors, turn down the block size
975 while True:
976 cmd = ['changes']
978 if block_size:
979 end = min(changeEnd, changeStart + block_size)
980 revisionRange = "%d,%d" % (changeStart, end)
981 else:
982 revisionRange = "%s,%s" % (changeStart, changeEnd)
984 for p in depotPaths:
985 cmd += ["%s...@%s" % (p, revisionRange)]
987 # fetch the changes
988 try:
989 result = p4CmdList(cmd, errors_as_exceptions=True)
990 except P4RequestSizeException as e:
991 if not block_size:
992 block_size = e.limit
993 elif block_size > e.limit:
994 block_size = e.limit
995 else:
996 block_size = max(2, block_size // 2)
998 if verbose: print("block size error, retrying with block size {0}".format(block_size))
999 continue
1000 except P4Exception as e:
1001 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1003 # Insert changes in chronological order
1004 for entry in reversed(result):
1005 if not entry.has_key('change'):
1006 continue
1007 changes.add(int(entry['change']))
1009 if not block_size:
1010 break
1012 if end >= changeEnd:
1013 break
1015 changeStart = end + 1
1017 changes = sorted(changes)
1018 return changes
1020 def p4PathStartsWith(path, prefix):
1021 # This method tries to remedy a potential mixed-case issue:
1023 # If UserA adds //depot/DirA/file1
1024 # and UserB adds //depot/dira/file2
1026 # we may or may not have a problem. If you have core.ignorecase=true,
1027 # we treat DirA and dira as the same directory
1028 if gitConfigBool("core.ignorecase"):
1029 return path.lower().startswith(prefix.lower())
1030 return path.startswith(prefix)
1032 def getClientSpec():
1033 """Look at the p4 client spec, create a View() object that contains
1034 all the mappings, and return it."""
1036 specList = p4CmdList("client -o")
1037 if len(specList) != 1:
1038 die('Output from "client -o" is %d lines, expecting 1' %
1039 len(specList))
1041 # dictionary of all client parameters
1042 entry = specList[0]
1044 # the //client/ name
1045 client_name = entry["Client"]
1047 # just the keys that start with "View"
1048 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1050 # hold this new View
1051 view = View(client_name)
1053 # append the lines, in order, to the view
1054 for view_num in range(len(view_keys)):
1055 k = "View%d" % view_num
1056 if k not in view_keys:
1057 die("Expected view key %s missing" % k)
1058 view.append(entry[k])
1060 return view
1062 def getClientRoot():
1063 """Grab the client directory."""
1065 output = p4CmdList("client -o")
1066 if len(output) != 1:
1067 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1069 entry = output[0]
1070 if "Root" not in entry:
1071 die('Client has no "Root"')
1073 return entry["Root"]
1076 # P4 wildcards are not allowed in filenames. P4 complains
1077 # if you simply add them, but you can force it with "-f", in
1078 # which case it translates them into %xx encoding internally.
1080 def wildcard_decode(path):
1081 # Search for and fix just these four characters. Do % last so
1082 # that fixing it does not inadvertently create new %-escapes.
1083 # Cannot have * in a filename in windows; untested as to
1084 # what p4 would do in such a case.
1085 if not platform.system() == "Windows":
1086 path = path.replace("%2A", "*")
1087 path = path.replace("%23", "#") \
1088 .replace("%40", "@") \
1089 .replace("%25", "%")
1090 return path
1092 def wildcard_encode(path):
1093 # do % first to avoid double-encoding the %s introduced here
1094 path = path.replace("%", "%25") \
1095 .replace("*", "%2A") \
1096 .replace("#", "%23") \
1097 .replace("@", "%40")
1098 return path
1100 def wildcard_present(path):
1101 m = re.search("[*#@%]", path)
1102 return m is not None
1104 class LargeFileSystem(object):
1105 """Base class for large file system support."""
1107 def __init__(self, writeToGitStream):
1108 self.largeFiles = set()
1109 self.writeToGitStream = writeToGitStream
1111 def generatePointer(self, cloneDestination, contentFile):
1112 """Return the content of a pointer file that is stored in Git instead of
1113 the actual content."""
1114 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1116 def pushFile(self, localLargeFile):
1117 """Push the actual content which is not stored in the Git repository to
1118 a server."""
1119 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1121 def hasLargeFileExtension(self, relPath):
1122 return reduce(
1123 lambda a, b: a or b,
1124 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1125 False
1128 def generateTempFile(self, contents):
1129 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1130 for d in contents:
1131 contentFile.write(d)
1132 contentFile.close()
1133 return contentFile.name
1135 def exceedsLargeFileThreshold(self, relPath, contents):
1136 if gitConfigInt('git-p4.largeFileThreshold'):
1137 contentsSize = sum(len(d) for d in contents)
1138 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1139 return True
1140 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1141 contentsSize = sum(len(d) for d in contents)
1142 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1143 return False
1144 contentTempFile = self.generateTempFile(contents)
1145 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1146 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1147 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1148 zf.close()
1149 compressedContentsSize = zf.infolist()[0].compress_size
1150 os.remove(contentTempFile)
1151 os.remove(compressedContentFile.name)
1152 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1153 return True
1154 return False
1156 def addLargeFile(self, relPath):
1157 self.largeFiles.add(relPath)
1159 def removeLargeFile(self, relPath):
1160 self.largeFiles.remove(relPath)
1162 def isLargeFile(self, relPath):
1163 return relPath in self.largeFiles
1165 def processContent(self, git_mode, relPath, contents):
1166 """Processes the content of git fast import. This method decides if a
1167 file is stored in the large file system and handles all necessary
1168 steps."""
1169 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1170 contentTempFile = self.generateTempFile(contents)
1171 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1172 if pointer_git_mode:
1173 git_mode = pointer_git_mode
1174 if localLargeFile:
1175 # Move temp file to final location in large file system
1176 largeFileDir = os.path.dirname(localLargeFile)
1177 if not os.path.isdir(largeFileDir):
1178 os.makedirs(largeFileDir)
1179 shutil.move(contentTempFile, localLargeFile)
1180 self.addLargeFile(relPath)
1181 if gitConfigBool('git-p4.largeFilePush'):
1182 self.pushFile(localLargeFile)
1183 if verbose:
1184 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1185 return (git_mode, contents)
1187 class MockLFS(LargeFileSystem):
1188 """Mock large file system for testing."""
1190 def generatePointer(self, contentFile):
1191 """The pointer content is the original content prefixed with "pointer-".
1192 The local filename of the large file storage is derived from the file content.
1194 with open(contentFile, 'r') as f:
1195 content = next(f)
1196 gitMode = '100644'
1197 pointerContents = 'pointer-' + content
1198 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1199 return (gitMode, pointerContents, localLargeFile)
1201 def pushFile(self, localLargeFile):
1202 """The remote filename of the large file storage is the same as the local
1203 one but in a different directory.
1205 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1206 if not os.path.exists(remotePath):
1207 os.makedirs(remotePath)
1208 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1210 class GitLFS(LargeFileSystem):
1211 """Git LFS as backend for the git-p4 large file system.
1212 See https://git-lfs.github.com/ for details."""
1214 def __init__(self, *args):
1215 LargeFileSystem.__init__(self, *args)
1216 self.baseGitAttributes = []
1218 def generatePointer(self, contentFile):
1219 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1220 mode and content which is stored in the Git repository instead of
1221 the actual content. Return also the new location of the actual
1222 content.
1224 if os.path.getsize(contentFile) == 0:
1225 return (None, '', None)
1227 pointerProcess = subprocess.Popen(
1228 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1229 stdout=subprocess.PIPE
1231 pointerFile = pointerProcess.stdout.read()
1232 if pointerProcess.wait():
1233 os.remove(contentFile)
1234 die('git-lfs pointer command failed. Did you install the extension?')
1236 # Git LFS removed the preamble in the output of the 'pointer' command
1237 # starting from version 1.2.0. Check for the preamble here to support
1238 # earlier versions.
1239 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1240 if pointerFile.startswith('Git LFS pointer for'):
1241 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1243 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1244 localLargeFile = os.path.join(
1245 os.getcwd(),
1246 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1247 oid,
1249 # LFS Spec states that pointer files should not have the executable bit set.
1250 gitMode = '100644'
1251 return (gitMode, pointerFile, localLargeFile)
1253 def pushFile(self, localLargeFile):
1254 uploadProcess = subprocess.Popen(
1255 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1257 if uploadProcess.wait():
1258 die('git-lfs push command failed. Did you define a remote?')
1260 def generateGitAttributes(self):
1261 return (
1262 self.baseGitAttributes +
1264 '\n',
1265 '#\n',
1266 '# Git LFS (see https://git-lfs.github.com/)\n',
1267 '#\n',
1269 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1270 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1272 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1273 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1277 def addLargeFile(self, relPath):
1278 LargeFileSystem.addLargeFile(self, relPath)
1279 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1281 def removeLargeFile(self, relPath):
1282 LargeFileSystem.removeLargeFile(self, relPath)
1283 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1285 def processContent(self, git_mode, relPath, contents):
1286 if relPath == '.gitattributes':
1287 self.baseGitAttributes = contents
1288 return (git_mode, self.generateGitAttributes())
1289 else:
1290 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1292 class Command:
1293 def __init__(self):
1294 self.usage = "usage: %prog [options]"
1295 self.needsGit = True
1296 self.verbose = False
1298 # This is required for the "append" cloneExclude action
1299 def ensure_value(self, attr, value):
1300 if not hasattr(self, attr) or getattr(self, attr) is None:
1301 setattr(self, attr, value)
1302 return getattr(self, attr)
1304 class P4UserMap:
1305 def __init__(self):
1306 self.userMapFromPerforceServer = False
1307 self.myP4UserId = None
1309 def p4UserId(self):
1310 if self.myP4UserId:
1311 return self.myP4UserId
1313 results = p4CmdList("user -o")
1314 for r in results:
1315 if r.has_key('User'):
1316 self.myP4UserId = r['User']
1317 return r['User']
1318 die("Could not find your p4 user id")
1320 def p4UserIsMe(self, p4User):
1321 # return True if the given p4 user is actually me
1322 me = self.p4UserId()
1323 if not p4User or p4User != me:
1324 return False
1325 else:
1326 return True
1328 def getUserCacheFilename(self):
1329 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1330 return home + "/.gitp4-usercache.txt"
1332 def getUserMapFromPerforceServer(self):
1333 if self.userMapFromPerforceServer:
1334 return
1335 self.users = {}
1336 self.emails = {}
1338 for output in p4CmdList("users"):
1339 if not output.has_key("User"):
1340 continue
1341 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1342 self.emails[output["Email"]] = output["User"]
1344 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1345 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1346 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1347 if mapUser and len(mapUser[0]) == 3:
1348 user = mapUser[0][0]
1349 fullname = mapUser[0][1]
1350 email = mapUser[0][2]
1351 self.users[user] = fullname + " <" + email + ">"
1352 self.emails[email] = user
1354 s = ''
1355 for (key, val) in self.users.items():
1356 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1358 open(self.getUserCacheFilename(), "wb").write(s)
1359 self.userMapFromPerforceServer = True
1361 def loadUserMapFromCache(self):
1362 self.users = {}
1363 self.userMapFromPerforceServer = False
1364 try:
1365 cache = open(self.getUserCacheFilename(), "rb")
1366 lines = cache.readlines()
1367 cache.close()
1368 for line in lines:
1369 entry = line.strip().split("\t")
1370 self.users[entry[0]] = entry[1]
1371 except IOError:
1372 self.getUserMapFromPerforceServer()
1374 class P4Debug(Command):
1375 def __init__(self):
1376 Command.__init__(self)
1377 self.options = []
1378 self.description = "A tool to debug the output of p4 -G."
1379 self.needsGit = False
1381 def run(self, args):
1382 j = 0
1383 for output in p4CmdList(args):
1384 print 'Element: %d' % j
1385 j += 1
1386 print output
1387 return True
1389 class P4RollBack(Command):
1390 def __init__(self):
1391 Command.__init__(self)
1392 self.options = [
1393 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1395 self.description = "A tool to debug the multi-branch import. Don't use :)"
1396 self.rollbackLocalBranches = False
1398 def run(self, args):
1399 if len(args) != 1:
1400 return False
1401 maxChange = int(args[0])
1403 if "p4ExitCode" in p4Cmd("changes -m 1"):
1404 die("Problems executing p4");
1406 if self.rollbackLocalBranches:
1407 refPrefix = "refs/heads/"
1408 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1409 else:
1410 refPrefix = "refs/remotes/"
1411 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1413 for line in lines:
1414 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1415 line = line.strip()
1416 ref = refPrefix + line
1417 log = extractLogMessageFromGitCommit(ref)
1418 settings = extractSettingsGitLog(log)
1420 depotPaths = settings['depot-paths']
1421 change = settings['change']
1423 changed = False
1425 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1426 for p in depotPaths]))) == 0:
1427 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1428 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1429 continue
1431 while change and int(change) > maxChange:
1432 changed = True
1433 if self.verbose:
1434 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1435 system("git update-ref %s \"%s^\"" % (ref, ref))
1436 log = extractLogMessageFromGitCommit(ref)
1437 settings = extractSettingsGitLog(log)
1440 depotPaths = settings['depot-paths']
1441 change = settings['change']
1443 if changed:
1444 print "%s rewound to %s" % (ref, change)
1446 return True
1448 class P4Submit(Command, P4UserMap):
1450 conflict_behavior_choices = ("ask", "skip", "quit")
1452 def __init__(self):
1453 Command.__init__(self)
1454 P4UserMap.__init__(self)
1455 self.options = [
1456 optparse.make_option("--origin", dest="origin"),
1457 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1458 # preserve the user, requires relevant p4 permissions
1459 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1460 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1461 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1462 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1463 optparse.make_option("--conflict", dest="conflict_behavior",
1464 choices=self.conflict_behavior_choices),
1465 optparse.make_option("--branch", dest="branch"),
1466 optparse.make_option("--shelve", dest="shelve", action="store_true",
1467 help="Shelve instead of submit. Shelved files are reverted, "
1468 "restoring the workspace to the state before the shelve"),
1469 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1470 metavar="CHANGELIST",
1471 help="update an existing shelved changelist, implies --shelve, "
1472 "repeat in-order for multiple shelved changelists"),
1473 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1474 help="submit only the specified commit(s), one commit or xxx..xxx"),
1475 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1476 help="Disable rebase after submit is completed. Can be useful if you "
1477 "work from a local git branch that is not master"),
1478 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1479 help="Skip Perforce sync of p4/master after submit or shelve"),
1481 self.description = "Submit changes from git to the perforce depot."
1482 self.usage += " [name of git branch to submit into perforce depot]"
1483 self.origin = ""
1484 self.detectRenames = False
1485 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1486 self.dry_run = False
1487 self.shelve = False
1488 self.update_shelve = list()
1489 self.commit = ""
1490 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1491 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1492 self.prepare_p4_only = False
1493 self.conflict_behavior = None
1494 self.isWindows = (platform.system() == "Windows")
1495 self.exportLabels = False
1496 self.p4HasMoveCommand = p4_has_move_command()
1497 self.branch = None
1499 if gitConfig('git-p4.largeFileSystem'):
1500 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1502 def check(self):
1503 if len(p4CmdList("opened ...")) > 0:
1504 die("You have files opened with perforce! Close them before starting the sync.")
1506 def separate_jobs_from_description(self, message):
1507 """Extract and return a possible Jobs field in the commit
1508 message. It goes into a separate section in the p4 change
1509 specification.
1511 A jobs line starts with "Jobs:" and looks like a new field
1512 in a form. Values are white-space separated on the same
1513 line or on following lines that start with a tab.
1515 This does not parse and extract the full git commit message
1516 like a p4 form. It just sees the Jobs: line as a marker
1517 to pass everything from then on directly into the p4 form,
1518 but outside the description section.
1520 Return a tuple (stripped log message, jobs string)."""
1522 m = re.search(r'^Jobs:', message, re.MULTILINE)
1523 if m is None:
1524 return (message, None)
1526 jobtext = message[m.start():]
1527 stripped_message = message[:m.start()].rstrip()
1528 return (stripped_message, jobtext)
1530 def prepareLogMessage(self, template, message, jobs):
1531 """Edits the template returned from "p4 change -o" to insert
1532 the message in the Description field, and the jobs text in
1533 the Jobs field."""
1534 result = ""
1536 inDescriptionSection = False
1538 for line in template.split("\n"):
1539 if line.startswith("#"):
1540 result += line + "\n"
1541 continue
1543 if inDescriptionSection:
1544 if line.startswith("Files:") or line.startswith("Jobs:"):
1545 inDescriptionSection = False
1546 # insert Jobs section
1547 if jobs:
1548 result += jobs + "\n"
1549 else:
1550 continue
1551 else:
1552 if line.startswith("Description:"):
1553 inDescriptionSection = True
1554 line += "\n"
1555 for messageLine in message.split("\n"):
1556 line += "\t" + messageLine + "\n"
1558 result += line + "\n"
1560 return result
1562 def patchRCSKeywords(self, file, pattern):
1563 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1564 (handle, outFileName) = tempfile.mkstemp(dir='.')
1565 try:
1566 outFile = os.fdopen(handle, "w+")
1567 inFile = open(file, "r")
1568 regexp = re.compile(pattern, re.VERBOSE)
1569 for line in inFile.readlines():
1570 line = regexp.sub(r'$\1$', line)
1571 outFile.write(line)
1572 inFile.close()
1573 outFile.close()
1574 # Forcibly overwrite the original file
1575 os.unlink(file)
1576 shutil.move(outFileName, file)
1577 except:
1578 # cleanup our temporary file
1579 os.unlink(outFileName)
1580 print "Failed to strip RCS keywords in %s" % file
1581 raise
1583 print "Patched up RCS keywords in %s" % file
1585 def p4UserForCommit(self,id):
1586 # Return the tuple (perforce user,git email) for a given git commit id
1587 self.getUserMapFromPerforceServer()
1588 gitEmail = read_pipe(["git", "log", "--max-count=1",
1589 "--format=%ae", id])
1590 gitEmail = gitEmail.strip()
1591 if not self.emails.has_key(gitEmail):
1592 return (None,gitEmail)
1593 else:
1594 return (self.emails[gitEmail],gitEmail)
1596 def checkValidP4Users(self,commits):
1597 # check if any git authors cannot be mapped to p4 users
1598 for id in commits:
1599 (user,email) = self.p4UserForCommit(id)
1600 if not user:
1601 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1602 if gitConfigBool("git-p4.allowMissingP4Users"):
1603 print "%s" % msg
1604 else:
1605 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1607 def lastP4Changelist(self):
1608 # Get back the last changelist number submitted in this client spec. This
1609 # then gets used to patch up the username in the change. If the same
1610 # client spec is being used by multiple processes then this might go
1611 # wrong.
1612 results = p4CmdList("client -o") # find the current client
1613 client = None
1614 for r in results:
1615 if r.has_key('Client'):
1616 client = r['Client']
1617 break
1618 if not client:
1619 die("could not get client spec")
1620 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1621 for r in results:
1622 if r.has_key('change'):
1623 return r['change']
1624 die("Could not get changelist number for last submit - cannot patch up user details")
1626 def modifyChangelistUser(self, changelist, newUser):
1627 # fixup the user field of a changelist after it has been submitted.
1628 changes = p4CmdList("change -o %s" % changelist)
1629 if len(changes) != 1:
1630 die("Bad output from p4 change modifying %s to user %s" %
1631 (changelist, newUser))
1633 c = changes[0]
1634 if c['User'] == newUser: return # nothing to do
1635 c['User'] = newUser
1636 input = marshal.dumps(c)
1638 result = p4CmdList("change -f -i", stdin=input)
1639 for r in result:
1640 if r.has_key('code'):
1641 if r['code'] == 'error':
1642 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1643 if r.has_key('data'):
1644 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1645 return
1646 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1648 def canChangeChangelists(self):
1649 # check to see if we have p4 admin or super-user permissions, either of
1650 # which are required to modify changelists.
1651 results = p4CmdList(["protects", self.depotPath])
1652 for r in results:
1653 if r.has_key('perm'):
1654 if r['perm'] == 'admin':
1655 return 1
1656 if r['perm'] == 'super':
1657 return 1
1658 return 0
1660 def prepareSubmitTemplate(self, changelist=None):
1661 """Run "p4 change -o" to grab a change specification template.
1662 This does not use "p4 -G", as it is nice to keep the submission
1663 template in original order, since a human might edit it.
1665 Remove lines in the Files section that show changes to files
1666 outside the depot path we're committing into."""
1668 [upstream, settings] = findUpstreamBranchPoint()
1670 template = """\
1671 # A Perforce Change Specification.
1673 # Change: The change number. 'new' on a new changelist.
1674 # Date: The date this specification was last modified.
1675 # Client: The client on which the changelist was created. Read-only.
1676 # User: The user who created the changelist.
1677 # Status: Either 'pending' or 'submitted'. Read-only.
1678 # Type: Either 'public' or 'restricted'. Default is 'public'.
1679 # Description: Comments about the changelist. Required.
1680 # Jobs: What opened jobs are to be closed by this changelist.
1681 # You may delete jobs from this list. (New changelists only.)
1682 # Files: What opened files from the default changelist are to be added
1683 # to this changelist. You may delete files from this list.
1684 # (New changelists only.)
1686 files_list = []
1687 inFilesSection = False
1688 change_entry = None
1689 args = ['change', '-o']
1690 if changelist:
1691 args.append(str(changelist))
1692 for entry in p4CmdList(args):
1693 if not entry.has_key('code'):
1694 continue
1695 if entry['code'] == 'stat':
1696 change_entry = entry
1697 break
1698 if not change_entry:
1699 die('Failed to decode output of p4 change -o')
1700 for key, value in change_entry.iteritems():
1701 if key.startswith('File'):
1702 if settings.has_key('depot-paths'):
1703 if not [p for p in settings['depot-paths']
1704 if p4PathStartsWith(value, p)]:
1705 continue
1706 else:
1707 if not p4PathStartsWith(value, self.depotPath):
1708 continue
1709 files_list.append(value)
1710 continue
1711 # Output in the order expected by prepareLogMessage
1712 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1713 if not change_entry.has_key(key):
1714 continue
1715 template += '\n'
1716 template += key + ':'
1717 if key == 'Description':
1718 template += '\n'
1719 for field_line in change_entry[key].splitlines():
1720 template += '\t'+field_line+'\n'
1721 if len(files_list) > 0:
1722 template += '\n'
1723 template += 'Files:\n'
1724 for path in files_list:
1725 template += '\t'+path+'\n'
1726 return template
1728 def edit_template(self, template_file):
1729 """Invoke the editor to let the user change the submission
1730 message. Return true if okay to continue with the submit."""
1732 # if configured to skip the editing part, just submit
1733 if gitConfigBool("git-p4.skipSubmitEdit"):
1734 return True
1736 # look at the modification time, to check later if the user saved
1737 # the file
1738 mtime = os.stat(template_file).st_mtime
1740 # invoke the editor
1741 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1742 editor = os.environ.get("P4EDITOR")
1743 else:
1744 editor = read_pipe("git var GIT_EDITOR").strip()
1745 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1747 # If the file was not saved, prompt to see if this patch should
1748 # be skipped. But skip this verification step if configured so.
1749 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1750 return True
1752 # modification time updated means user saved the file
1753 if os.stat(template_file).st_mtime > mtime:
1754 return True
1756 while True:
1757 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1758 if response == 'y':
1759 return True
1760 if response == 'n':
1761 return False
1763 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1764 # diff
1765 if os.environ.has_key("P4DIFF"):
1766 del(os.environ["P4DIFF"])
1767 diff = ""
1768 for editedFile in editedFiles:
1769 diff += p4_read_pipe(['diff', '-du',
1770 wildcard_encode(editedFile)])
1772 # new file diff
1773 newdiff = ""
1774 for newFile in filesToAdd:
1775 newdiff += "==== new file ====\n"
1776 newdiff += "--- /dev/null\n"
1777 newdiff += "+++ %s\n" % newFile
1779 is_link = os.path.islink(newFile)
1780 expect_link = newFile in symlinks
1782 if is_link and expect_link:
1783 newdiff += "+%s\n" % os.readlink(newFile)
1784 else:
1785 f = open(newFile, "r")
1786 for line in f.readlines():
1787 newdiff += "+" + line
1788 f.close()
1790 return (diff + newdiff).replace('\r\n', '\n')
1792 def applyCommit(self, id):
1793 """Apply one commit, return True if it succeeded."""
1795 print "Applying", read_pipe(["git", "show", "-s",
1796 "--format=format:%h %s", id])
1798 (p4User, gitEmail) = self.p4UserForCommit(id)
1800 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1801 filesToAdd = set()
1802 filesToChangeType = set()
1803 filesToDelete = set()
1804 editedFiles = set()
1805 pureRenameCopy = set()
1806 symlinks = set()
1807 filesToChangeExecBit = {}
1808 all_files = list()
1810 for line in diff:
1811 diff = parseDiffTreeEntry(line)
1812 modifier = diff['status']
1813 path = diff['src']
1814 all_files.append(path)
1816 if modifier == "M":
1817 p4_edit(path)
1818 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1819 filesToChangeExecBit[path] = diff['dst_mode']
1820 editedFiles.add(path)
1821 elif modifier == "A":
1822 filesToAdd.add(path)
1823 filesToChangeExecBit[path] = diff['dst_mode']
1824 if path in filesToDelete:
1825 filesToDelete.remove(path)
1827 dst_mode = int(diff['dst_mode'], 8)
1828 if dst_mode == 0120000:
1829 symlinks.add(path)
1831 elif modifier == "D":
1832 filesToDelete.add(path)
1833 if path in filesToAdd:
1834 filesToAdd.remove(path)
1835 elif modifier == "C":
1836 src, dest = diff['src'], diff['dst']
1837 p4_integrate(src, dest)
1838 pureRenameCopy.add(dest)
1839 if diff['src_sha1'] != diff['dst_sha1']:
1840 p4_edit(dest)
1841 pureRenameCopy.discard(dest)
1842 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1843 p4_edit(dest)
1844 pureRenameCopy.discard(dest)
1845 filesToChangeExecBit[dest] = diff['dst_mode']
1846 if self.isWindows:
1847 # turn off read-only attribute
1848 os.chmod(dest, stat.S_IWRITE)
1849 os.unlink(dest)
1850 editedFiles.add(dest)
1851 elif modifier == "R":
1852 src, dest = diff['src'], diff['dst']
1853 if self.p4HasMoveCommand:
1854 p4_edit(src) # src must be open before move
1855 p4_move(src, dest) # opens for (move/delete, move/add)
1856 else:
1857 p4_integrate(src, dest)
1858 if diff['src_sha1'] != diff['dst_sha1']:
1859 p4_edit(dest)
1860 else:
1861 pureRenameCopy.add(dest)
1862 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1863 if not self.p4HasMoveCommand:
1864 p4_edit(dest) # with move: already open, writable
1865 filesToChangeExecBit[dest] = diff['dst_mode']
1866 if not self.p4HasMoveCommand:
1867 if self.isWindows:
1868 os.chmod(dest, stat.S_IWRITE)
1869 os.unlink(dest)
1870 filesToDelete.add(src)
1871 editedFiles.add(dest)
1872 elif modifier == "T":
1873 filesToChangeType.add(path)
1874 else:
1875 die("unknown modifier %s for %s" % (modifier, path))
1877 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1878 patchcmd = diffcmd + " | git apply "
1879 tryPatchCmd = patchcmd + "--check -"
1880 applyPatchCmd = patchcmd + "--check --apply -"
1881 patch_succeeded = True
1883 if os.system(tryPatchCmd) != 0:
1884 fixed_rcs_keywords = False
1885 patch_succeeded = False
1886 print "Unfortunately applying the change failed!"
1888 # Patch failed, maybe it's just RCS keyword woes. Look through
1889 # the patch to see if that's possible.
1890 if gitConfigBool("git-p4.attemptRCSCleanup"):
1891 file = None
1892 pattern = None
1893 kwfiles = {}
1894 for file in editedFiles | filesToDelete:
1895 # did this file's delta contain RCS keywords?
1896 pattern = p4_keywords_regexp_for_file(file)
1898 if pattern:
1899 # this file is a possibility...look for RCS keywords.
1900 regexp = re.compile(pattern, re.VERBOSE)
1901 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1902 if regexp.search(line):
1903 if verbose:
1904 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1905 kwfiles[file] = pattern
1906 break
1908 for file in kwfiles:
1909 if verbose:
1910 print "zapping %s with %s" % (line,pattern)
1911 # File is being deleted, so not open in p4. Must
1912 # disable the read-only bit on windows.
1913 if self.isWindows and file not in editedFiles:
1914 os.chmod(file, stat.S_IWRITE)
1915 self.patchRCSKeywords(file, kwfiles[file])
1916 fixed_rcs_keywords = True
1918 if fixed_rcs_keywords:
1919 print "Retrying the patch with RCS keywords cleaned up"
1920 if os.system(tryPatchCmd) == 0:
1921 patch_succeeded = True
1923 if not patch_succeeded:
1924 for f in editedFiles:
1925 p4_revert(f)
1926 return False
1929 # Apply the patch for real, and do add/delete/+x handling.
1931 system(applyPatchCmd)
1933 for f in filesToChangeType:
1934 p4_edit(f, "-t", "auto")
1935 for f in filesToAdd:
1936 p4_add(f)
1937 for f in filesToDelete:
1938 p4_revert(f)
1939 p4_delete(f)
1941 # Set/clear executable bits
1942 for f in filesToChangeExecBit.keys():
1943 mode = filesToChangeExecBit[f]
1944 setP4ExecBit(f, mode)
1946 update_shelve = 0
1947 if len(self.update_shelve) > 0:
1948 update_shelve = self.update_shelve.pop(0)
1949 p4_reopen_in_change(update_shelve, all_files)
1952 # Build p4 change description, starting with the contents
1953 # of the git commit message.
1955 logMessage = extractLogMessageFromGitCommit(id)
1956 logMessage = logMessage.strip()
1957 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1959 template = self.prepareSubmitTemplate(update_shelve)
1960 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1962 if self.preserveUser:
1963 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1965 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1966 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1967 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1968 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1970 separatorLine = "######## everything below this line is just the diff #######\n"
1971 if not self.prepare_p4_only:
1972 submitTemplate += separatorLine
1973 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1975 (handle, fileName) = tempfile.mkstemp()
1976 tmpFile = os.fdopen(handle, "w+b")
1977 if self.isWindows:
1978 submitTemplate = submitTemplate.replace("\n", "\r\n")
1979 tmpFile.write(submitTemplate)
1980 tmpFile.close()
1982 if self.prepare_p4_only:
1984 # Leave the p4 tree prepared, and the submit template around
1985 # and let the user decide what to do next
1987 print
1988 print "P4 workspace prepared for submission."
1989 print "To submit or revert, go to client workspace"
1990 print " " + self.clientPath
1991 print
1992 print "To submit, use \"p4 submit\" to write a new description,"
1993 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1994 " \"git p4\"." % fileName
1995 print "You can delete the file \"%s\" when finished." % fileName
1997 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1998 print "To preserve change ownership by user %s, you must\n" \
1999 "do \"p4 change -f <change>\" after submitting and\n" \
2000 "edit the User field."
2001 if pureRenameCopy:
2002 print "After submitting, renamed files must be re-synced."
2003 print "Invoke \"p4 sync -f\" on each of these files:"
2004 for f in pureRenameCopy:
2005 print " " + f
2007 print
2008 print "To revert the changes, use \"p4 revert ...\", and delete"
2009 print "the submit template file \"%s\"" % fileName
2010 if filesToAdd:
2011 print "Since the commit adds new files, they must be deleted:"
2012 for f in filesToAdd:
2013 print " " + f
2014 print
2015 return True
2018 # Let the user edit the change description, then submit it.
2020 submitted = False
2022 try:
2023 if self.edit_template(fileName):
2024 # read the edited message and submit
2025 tmpFile = open(fileName, "rb")
2026 message = tmpFile.read()
2027 tmpFile.close()
2028 if self.isWindows:
2029 message = message.replace("\r\n", "\n")
2030 submitTemplate = message[:message.index(separatorLine)]
2032 if update_shelve:
2033 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2034 elif self.shelve:
2035 p4_write_pipe(['shelve', '-i'], submitTemplate)
2036 else:
2037 p4_write_pipe(['submit', '-i'], submitTemplate)
2038 # The rename/copy happened by applying a patch that created a
2039 # new file. This leaves it writable, which confuses p4.
2040 for f in pureRenameCopy:
2041 p4_sync(f, "-f")
2043 if self.preserveUser:
2044 if p4User:
2045 # Get last changelist number. Cannot easily get it from
2046 # the submit command output as the output is
2047 # unmarshalled.
2048 changelist = self.lastP4Changelist()
2049 self.modifyChangelistUser(changelist, p4User)
2051 submitted = True
2053 finally:
2054 # skip this patch
2055 if not submitted or self.shelve:
2056 if self.shelve:
2057 print ("Reverting shelved files.")
2058 else:
2059 print ("Submission cancelled, undoing p4 changes.")
2060 for f in editedFiles | filesToDelete:
2061 p4_revert(f)
2062 for f in filesToAdd:
2063 p4_revert(f)
2064 os.remove(f)
2066 os.remove(fileName)
2067 return submitted
2069 # Export git tags as p4 labels. Create a p4 label and then tag
2070 # with that.
2071 def exportGitTags(self, gitTags):
2072 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2073 if len(validLabelRegexp) == 0:
2074 validLabelRegexp = defaultLabelRegexp
2075 m = re.compile(validLabelRegexp)
2077 for name in gitTags:
2079 if not m.match(name):
2080 if verbose:
2081 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
2082 continue
2084 # Get the p4 commit this corresponds to
2085 logMessage = extractLogMessageFromGitCommit(name)
2086 values = extractSettingsGitLog(logMessage)
2088 if not values.has_key('change'):
2089 # a tag pointing to something not sent to p4; ignore
2090 if verbose:
2091 print "git tag %s does not give a p4 commit" % name
2092 continue
2093 else:
2094 changelist = values['change']
2096 # Get the tag details.
2097 inHeader = True
2098 isAnnotated = False
2099 body = []
2100 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2101 l = l.strip()
2102 if inHeader:
2103 if re.match(r'tag\s+', l):
2104 isAnnotated = True
2105 elif re.match(r'\s*$', l):
2106 inHeader = False
2107 continue
2108 else:
2109 body.append(l)
2111 if not isAnnotated:
2112 body = ["lightweight tag imported by git p4\n"]
2114 # Create the label - use the same view as the client spec we are using
2115 clientSpec = getClientSpec()
2117 labelTemplate = "Label: %s\n" % name
2118 labelTemplate += "Description:\n"
2119 for b in body:
2120 labelTemplate += "\t" + b + "\n"
2121 labelTemplate += "View:\n"
2122 for depot_side in clientSpec.mappings:
2123 labelTemplate += "\t%s\n" % depot_side
2125 if self.dry_run:
2126 print "Would create p4 label %s for tag" % name
2127 elif self.prepare_p4_only:
2128 print "Not creating p4 label %s for tag due to option" \
2129 " --prepare-p4-only" % name
2130 else:
2131 p4_write_pipe(["label", "-i"], labelTemplate)
2133 # Use the label
2134 p4_system(["tag", "-l", name] +
2135 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2137 if verbose:
2138 print "created p4 label for tag %s" % name
2140 def run(self, args):
2141 if len(args) == 0:
2142 self.master = currentGitBranch()
2143 elif len(args) == 1:
2144 self.master = args[0]
2145 if not branchExists(self.master):
2146 die("Branch %s does not exist" % self.master)
2147 else:
2148 return False
2150 for i in self.update_shelve:
2151 if i <= 0:
2152 sys.exit("invalid changelist %d" % i)
2154 if self.master:
2155 allowSubmit = gitConfig("git-p4.allowSubmit")
2156 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2157 die("%s is not in git-p4.allowSubmit" % self.master)
2159 [upstream, settings] = findUpstreamBranchPoint()
2160 self.depotPath = settings['depot-paths'][0]
2161 if len(self.origin) == 0:
2162 self.origin = upstream
2164 if len(self.update_shelve) > 0:
2165 self.shelve = True
2167 if self.preserveUser:
2168 if not self.canChangeChangelists():
2169 die("Cannot preserve user names without p4 super-user or admin permissions")
2171 # if not set from the command line, try the config file
2172 if self.conflict_behavior is None:
2173 val = gitConfig("git-p4.conflict")
2174 if val:
2175 if val not in self.conflict_behavior_choices:
2176 die("Invalid value '%s' for config git-p4.conflict" % val)
2177 else:
2178 val = "ask"
2179 self.conflict_behavior = val
2181 if self.verbose:
2182 print "Origin branch is " + self.origin
2184 if len(self.depotPath) == 0:
2185 print "Internal error: cannot locate perforce depot path from existing branches"
2186 sys.exit(128)
2188 self.useClientSpec = False
2189 if gitConfigBool("git-p4.useclientspec"):
2190 self.useClientSpec = True
2191 if self.useClientSpec:
2192 self.clientSpecDirs = getClientSpec()
2194 # Check for the existence of P4 branches
2195 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2197 if self.useClientSpec and not branchesDetected:
2198 # all files are relative to the client spec
2199 self.clientPath = getClientRoot()
2200 else:
2201 self.clientPath = p4Where(self.depotPath)
2203 if self.clientPath == "":
2204 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2206 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2207 self.oldWorkingDirectory = os.getcwd()
2209 # ensure the clientPath exists
2210 new_client_dir = False
2211 if not os.path.exists(self.clientPath):
2212 new_client_dir = True
2213 os.makedirs(self.clientPath)
2215 chdir(self.clientPath, is_client_path=True)
2216 if self.dry_run:
2217 print "Would synchronize p4 checkout in %s" % self.clientPath
2218 else:
2219 print "Synchronizing p4 checkout..."
2220 if new_client_dir:
2221 # old one was destroyed, and maybe nobody told p4
2222 p4_sync("...", "-f")
2223 else:
2224 p4_sync("...")
2225 self.check()
2227 commits = []
2228 if self.master:
2229 committish = self.master
2230 else:
2231 committish = 'HEAD'
2233 if self.commit != "":
2234 if self.commit.find("..") != -1:
2235 limits_ish = self.commit.split("..")
2236 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2237 commits.append(line.strip())
2238 commits.reverse()
2239 else:
2240 commits.append(self.commit)
2241 else:
2242 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2243 commits.append(line.strip())
2244 commits.reverse()
2246 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2247 self.checkAuthorship = False
2248 else:
2249 self.checkAuthorship = True
2251 if self.preserveUser:
2252 self.checkValidP4Users(commits)
2255 # Build up a set of options to be passed to diff when
2256 # submitting each commit to p4.
2258 if self.detectRenames:
2259 # command-line -M arg
2260 self.diffOpts = "-M"
2261 else:
2262 # If not explicitly set check the config variable
2263 detectRenames = gitConfig("git-p4.detectRenames")
2265 if detectRenames.lower() == "false" or detectRenames == "":
2266 self.diffOpts = ""
2267 elif detectRenames.lower() == "true":
2268 self.diffOpts = "-M"
2269 else:
2270 self.diffOpts = "-M%s" % detectRenames
2272 # no command-line arg for -C or --find-copies-harder, just
2273 # config variables
2274 detectCopies = gitConfig("git-p4.detectCopies")
2275 if detectCopies.lower() == "false" or detectCopies == "":
2276 pass
2277 elif detectCopies.lower() == "true":
2278 self.diffOpts += " -C"
2279 else:
2280 self.diffOpts += " -C%s" % detectCopies
2282 if gitConfigBool("git-p4.detectCopiesHarder"):
2283 self.diffOpts += " --find-copies-harder"
2285 num_shelves = len(self.update_shelve)
2286 if num_shelves > 0 and num_shelves != len(commits):
2287 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2288 (len(commits), num_shelves))
2291 # Apply the commits, one at a time. On failure, ask if should
2292 # continue to try the rest of the patches, or quit.
2294 if self.dry_run:
2295 print "Would apply"
2296 applied = []
2297 last = len(commits) - 1
2298 for i, commit in enumerate(commits):
2299 if self.dry_run:
2300 print " ", read_pipe(["git", "show", "-s",
2301 "--format=format:%h %s", commit])
2302 ok = True
2303 else:
2304 ok = self.applyCommit(commit)
2305 if ok:
2306 applied.append(commit)
2307 else:
2308 if self.prepare_p4_only and i < last:
2309 print "Processing only the first commit due to option" \
2310 " --prepare-p4-only"
2311 break
2312 if i < last:
2313 quit = False
2314 while True:
2315 # prompt for what to do, or use the option/variable
2316 if self.conflict_behavior == "ask":
2317 print "What do you want to do?"
2318 response = raw_input("[s]kip this commit but apply"
2319 " the rest, or [q]uit? ")
2320 if not response:
2321 continue
2322 elif self.conflict_behavior == "skip":
2323 response = "s"
2324 elif self.conflict_behavior == "quit":
2325 response = "q"
2326 else:
2327 die("Unknown conflict_behavior '%s'" %
2328 self.conflict_behavior)
2330 if response[0] == "s":
2331 print "Skipping this commit, but applying the rest"
2332 break
2333 if response[0] == "q":
2334 print "Quitting"
2335 quit = True
2336 break
2337 if quit:
2338 break
2340 chdir(self.oldWorkingDirectory)
2341 shelved_applied = "shelved" if self.shelve else "applied"
2342 if self.dry_run:
2343 pass
2344 elif self.prepare_p4_only:
2345 pass
2346 elif len(commits) == len(applied):
2347 print ("All commits {0}!".format(shelved_applied))
2349 sync = P4Sync()
2350 if self.branch:
2351 sync.branch = self.branch
2352 if self.disable_p4sync:
2353 sync.sync_origin_only()
2354 else:
2355 sync.run([])
2357 if not self.disable_rebase:
2358 rebase = P4Rebase()
2359 rebase.rebase()
2361 else:
2362 if len(applied) == 0:
2363 print ("No commits {0}.".format(shelved_applied))
2364 else:
2365 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2366 for c in commits:
2367 if c in applied:
2368 star = "*"
2369 else:
2370 star = " "
2371 print star, read_pipe(["git", "show", "-s",
2372 "--format=format:%h %s", c])
2373 print "You will have to do 'git p4 sync' and rebase."
2375 if gitConfigBool("git-p4.exportLabels"):
2376 self.exportLabels = True
2378 if self.exportLabels:
2379 p4Labels = getP4Labels(self.depotPath)
2380 gitTags = getGitTags()
2382 missingGitTags = gitTags - p4Labels
2383 self.exportGitTags(missingGitTags)
2385 # exit with error unless everything applied perfectly
2386 if len(commits) != len(applied):
2387 sys.exit(1)
2389 return True
2391 class View(object):
2392 """Represent a p4 view ("p4 help views"), and map files in a
2393 repo according to the view."""
2395 def __init__(self, client_name):
2396 self.mappings = []
2397 self.client_prefix = "//%s/" % client_name
2398 # cache results of "p4 where" to lookup client file locations
2399 self.client_spec_path_cache = {}
2401 def append(self, view_line):
2402 """Parse a view line, splitting it into depot and client
2403 sides. Append to self.mappings, preserving order. This
2404 is only needed for tag creation."""
2406 # Split the view line into exactly two words. P4 enforces
2407 # structure on these lines that simplifies this quite a bit.
2409 # Either or both words may be double-quoted.
2410 # Single quotes do not matter.
2411 # Double-quote marks cannot occur inside the words.
2412 # A + or - prefix is also inside the quotes.
2413 # There are no quotes unless they contain a space.
2414 # The line is already white-space stripped.
2415 # The two words are separated by a single space.
2417 if view_line[0] == '"':
2418 # First word is double quoted. Find its end.
2419 close_quote_index = view_line.find('"', 1)
2420 if close_quote_index <= 0:
2421 die("No first-word closing quote found: %s" % view_line)
2422 depot_side = view_line[1:close_quote_index]
2423 # skip closing quote and space
2424 rhs_index = close_quote_index + 1 + 1
2425 else:
2426 space_index = view_line.find(" ")
2427 if space_index <= 0:
2428 die("No word-splitting space found: %s" % view_line)
2429 depot_side = view_line[0:space_index]
2430 rhs_index = space_index + 1
2432 # prefix + means overlay on previous mapping
2433 if depot_side.startswith("+"):
2434 depot_side = depot_side[1:]
2436 # prefix - means exclude this path, leave out of mappings
2437 exclude = False
2438 if depot_side.startswith("-"):
2439 exclude = True
2440 depot_side = depot_side[1:]
2442 if not exclude:
2443 self.mappings.append(depot_side)
2445 def convert_client_path(self, clientFile):
2446 # chop off //client/ part to make it relative
2447 if not clientFile.startswith(self.client_prefix):
2448 die("No prefix '%s' on clientFile '%s'" %
2449 (self.client_prefix, clientFile))
2450 return clientFile[len(self.client_prefix):]
2452 def update_client_spec_path_cache(self, files):
2453 """ Caching file paths by "p4 where" batch query """
2455 # List depot file paths exclude that already cached
2456 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2458 if len(fileArgs) == 0:
2459 return # All files in cache
2461 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2462 for res in where_result:
2463 if "code" in res and res["code"] == "error":
2464 # assume error is "... file(s) not in client view"
2465 continue
2466 if "clientFile" not in res:
2467 die("No clientFile in 'p4 where' output")
2468 if "unmap" in res:
2469 # it will list all of them, but only one not unmap-ped
2470 continue
2471 if gitConfigBool("core.ignorecase"):
2472 res['depotFile'] = res['depotFile'].lower()
2473 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2475 # not found files or unmap files set to ""
2476 for depotFile in fileArgs:
2477 if gitConfigBool("core.ignorecase"):
2478 depotFile = depotFile.lower()
2479 if depotFile not in self.client_spec_path_cache:
2480 self.client_spec_path_cache[depotFile] = ""
2482 def map_in_client(self, depot_path):
2483 """Return the relative location in the client where this
2484 depot file should live. Returns "" if the file should
2485 not be mapped in the client."""
2487 if gitConfigBool("core.ignorecase"):
2488 depot_path = depot_path.lower()
2490 if depot_path in self.client_spec_path_cache:
2491 return self.client_spec_path_cache[depot_path]
2493 die( "Error: %s is not found in client spec path" % depot_path )
2494 return ""
2496 class P4Sync(Command, P4UserMap):
2497 delete_actions = ( "delete", "move/delete", "purge" )
2499 def __init__(self):
2500 Command.__init__(self)
2501 P4UserMap.__init__(self)
2502 self.options = [
2503 optparse.make_option("--branch", dest="branch"),
2504 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2505 optparse.make_option("--changesfile", dest="changesFile"),
2506 optparse.make_option("--silent", dest="silent", action="store_true"),
2507 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2508 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2509 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2510 help="Import into refs/heads/ , not refs/remotes"),
2511 optparse.make_option("--max-changes", dest="maxChanges",
2512 help="Maximum number of changes to import"),
2513 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2514 help="Internal block size to use when iteratively calling p4 changes"),
2515 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2516 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2517 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2518 help="Only sync files that are included in the Perforce Client Spec"),
2519 optparse.make_option("-/", dest="cloneExclude",
2520 action="append", type="string",
2521 help="exclude depot path"),
2523 self.description = """Imports from Perforce into a git repository.\n
2524 example:
2525 //depot/my/project/ -- to import the current head
2526 //depot/my/project/@all -- to import everything
2527 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2529 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2531 self.usage += " //depot/path[@revRange]"
2532 self.silent = False
2533 self.createdBranches = set()
2534 self.committedChanges = set()
2535 self.branch = ""
2536 self.detectBranches = False
2537 self.detectLabels = False
2538 self.importLabels = False
2539 self.changesFile = ""
2540 self.syncWithOrigin = True
2541 self.importIntoRemotes = True
2542 self.maxChanges = ""
2543 self.changes_block_size = None
2544 self.keepRepoPath = False
2545 self.depotPaths = None
2546 self.p4BranchesInGit = []
2547 self.cloneExclude = []
2548 self.useClientSpec = False
2549 self.useClientSpec_from_options = False
2550 self.clientSpecDirs = None
2551 self.tempBranches = []
2552 self.tempBranchLocation = "refs/git-p4-tmp"
2553 self.largeFileSystem = None
2554 self.suppress_meta_comment = False
2556 if gitConfig('git-p4.largeFileSystem'):
2557 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2558 self.largeFileSystem = largeFileSystemConstructor(
2559 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2562 if gitConfig("git-p4.syncFromOrigin") == "false":
2563 self.syncWithOrigin = False
2565 self.depotPaths = []
2566 self.changeRange = ""
2567 self.previousDepotPaths = []
2568 self.hasOrigin = False
2570 # map from branch depot path to parent branch
2571 self.knownBranches = {}
2572 self.initialParents = {}
2574 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2575 self.labels = {}
2577 # Force a checkpoint in fast-import and wait for it to finish
2578 def checkpoint(self):
2579 self.gitStream.write("checkpoint\n\n")
2580 self.gitStream.write("progress checkpoint\n\n")
2581 out = self.gitOutput.readline()
2582 if self.verbose:
2583 print "checkpoint finished: " + out
2585 def cmp_shelved(self, path, filerev, revision):
2586 """ Determine if a path at revision #filerev is the same as the file
2587 at revision @revision for a shelved changelist. If they don't match,
2588 unshelving won't be safe (we will get other changes mixed in).
2590 This is comparing the revision that the shelved changelist is *based* on, not
2591 the shelved changelist itself.
2593 ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)])
2594 if verbose:
2595 print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret))
2596 return ret["status"] == "identical"
2598 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):
2599 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2600 for path in self.cloneExclude]
2601 files = []
2602 fnum = 0
2603 while commit.has_key("depotFile%s" % fnum):
2604 path = commit["depotFile%s" % fnum]
2606 if [p for p in self.cloneExclude
2607 if p4PathStartsWith(path, p)]:
2608 found = False
2609 else:
2610 found = [p for p in self.depotPaths
2611 if p4PathStartsWith(path, p)]
2612 if not found:
2613 fnum = fnum + 1
2614 continue
2616 file = {}
2617 file["path"] = path
2618 file["rev"] = commit["rev%s" % fnum]
2619 file["action"] = commit["action%s" % fnum]
2620 file["type"] = commit["type%s" % fnum]
2621 if shelved:
2622 file["shelved_cl"] = int(shelved_cl)
2624 # For shelved changelists, check that the revision of each file that the
2625 # shelve was based on matches the revision that we are using for the
2626 # starting point for git-fast-import (self.initialParent). Otherwise
2627 # the resulting diff will contain deltas from multiple commits.
2629 if file["action"] != "add" and \
2630 not self.cmp_shelved(path, file["rev"], origin_revision):
2631 sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2632 commit["change"], self.initialParent, path))
2634 files.append(file)
2635 fnum = fnum + 1
2636 return files
2638 def extractJobsFromCommit(self, commit):
2639 jobs = []
2640 jnum = 0
2641 while commit.has_key("job%s" % jnum):
2642 job = commit["job%s" % jnum]
2643 jobs.append(job)
2644 jnum = jnum + 1
2645 return jobs
2647 def stripRepoPath(self, path, prefixes):
2648 """When streaming files, this is called to map a p4 depot path
2649 to where it should go in git. The prefixes are either
2650 self.depotPaths, or self.branchPrefixes in the case of
2651 branch detection."""
2653 if self.useClientSpec:
2654 # branch detection moves files up a level (the branch name)
2655 # from what client spec interpretation gives
2656 path = self.clientSpecDirs.map_in_client(path)
2657 if self.detectBranches:
2658 for b in self.knownBranches:
2659 if path.startswith(b + "/"):
2660 path = path[len(b)+1:]
2662 elif self.keepRepoPath:
2663 # Preserve everything in relative path name except leading
2664 # //depot/; just look at first prefix as they all should
2665 # be in the same depot.
2666 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2667 if p4PathStartsWith(path, depot):
2668 path = path[len(depot):]
2670 else:
2671 for p in prefixes:
2672 if p4PathStartsWith(path, p):
2673 path = path[len(p):]
2674 break
2676 path = wildcard_decode(path)
2677 return path
2679 def splitFilesIntoBranches(self, commit):
2680 """Look at each depotFile in the commit to figure out to what
2681 branch it belongs."""
2683 if self.clientSpecDirs:
2684 files = self.extractFilesFromCommit(commit)
2685 self.clientSpecDirs.update_client_spec_path_cache(files)
2687 branches = {}
2688 fnum = 0
2689 while commit.has_key("depotFile%s" % fnum):
2690 path = commit["depotFile%s" % fnum]
2691 found = [p for p in self.depotPaths
2692 if p4PathStartsWith(path, p)]
2693 if not found:
2694 fnum = fnum + 1
2695 continue
2697 file = {}
2698 file["path"] = path
2699 file["rev"] = commit["rev%s" % fnum]
2700 file["action"] = commit["action%s" % fnum]
2701 file["type"] = commit["type%s" % fnum]
2702 fnum = fnum + 1
2704 # start with the full relative path where this file would
2705 # go in a p4 client
2706 if self.useClientSpec:
2707 relPath = self.clientSpecDirs.map_in_client(path)
2708 else:
2709 relPath = self.stripRepoPath(path, self.depotPaths)
2711 for branch in self.knownBranches.keys():
2712 # add a trailing slash so that a commit into qt/4.2foo
2713 # doesn't end up in qt/4.2, e.g.
2714 if relPath.startswith(branch + "/"):
2715 if branch not in branches:
2716 branches[branch] = []
2717 branches[branch].append(file)
2718 break
2720 return branches
2722 def writeToGitStream(self, gitMode, relPath, contents):
2723 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2724 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2725 for d in contents:
2726 self.gitStream.write(d)
2727 self.gitStream.write('\n')
2729 def encodeWithUTF8(self, path):
2730 try:
2731 path.decode('ascii')
2732 except:
2733 encoding = 'utf8'
2734 if gitConfig('git-p4.pathEncoding'):
2735 encoding = gitConfig('git-p4.pathEncoding')
2736 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2737 if self.verbose:
2738 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2739 return path
2741 # output one file from the P4 stream
2742 # - helper for streamP4Files
2744 def streamOneP4File(self, file, contents):
2745 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2746 relPath = self.encodeWithUTF8(relPath)
2747 if verbose:
2748 size = int(self.stream_file['fileSize'])
2749 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2750 sys.stdout.flush()
2752 (type_base, type_mods) = split_p4_type(file["type"])
2754 git_mode = "100644"
2755 if "x" in type_mods:
2756 git_mode = "100755"
2757 if type_base == "symlink":
2758 git_mode = "120000"
2759 # p4 print on a symlink sometimes contains "target\n";
2760 # if it does, remove the newline
2761 data = ''.join(contents)
2762 if not data:
2763 # Some version of p4 allowed creating a symlink that pointed
2764 # to nothing. This causes p4 errors when checking out such
2765 # a change, and errors here too. Work around it by ignoring
2766 # the bad symlink; hopefully a future change fixes it.
2767 print "\nIgnoring empty symlink in %s" % file['depotFile']
2768 return
2769 elif data[-1] == '\n':
2770 contents = [data[:-1]]
2771 else:
2772 contents = [data]
2774 if type_base == "utf16":
2775 # p4 delivers different text in the python output to -G
2776 # than it does when using "print -o", or normal p4 client
2777 # operations. utf16 is converted to ascii or utf8, perhaps.
2778 # But ascii text saved as -t utf16 is completely mangled.
2779 # Invoke print -o to get the real contents.
2781 # On windows, the newlines will always be mangled by print, so put
2782 # them back too. This is not needed to the cygwin windows version,
2783 # just the native "NT" type.
2785 try:
2786 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2787 except Exception as e:
2788 if 'Translation of file content failed' in str(e):
2789 type_base = 'binary'
2790 else:
2791 raise e
2792 else:
2793 if p4_version_string().find('/NT') >= 0:
2794 text = text.replace('\r\n', '\n')
2795 contents = [ text ]
2797 if type_base == "apple":
2798 # Apple filetype files will be streamed as a concatenation of
2799 # its appledouble header and the contents. This is useless
2800 # on both macs and non-macs. If using "print -q -o xx", it
2801 # will create "xx" with the data, and "%xx" with the header.
2802 # This is also not very useful.
2804 # Ideally, someday, this script can learn how to generate
2805 # appledouble files directly and import those to git, but
2806 # non-mac machines can never find a use for apple filetype.
2807 print "\nIgnoring apple filetype file %s" % file['depotFile']
2808 return
2810 # Note that we do not try to de-mangle keywords on utf16 files,
2811 # even though in theory somebody may want that.
2812 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2813 if pattern:
2814 regexp = re.compile(pattern, re.VERBOSE)
2815 text = ''.join(contents)
2816 text = regexp.sub(r'$\1$', text)
2817 contents = [ text ]
2819 if self.largeFileSystem:
2820 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2822 self.writeToGitStream(git_mode, relPath, contents)
2824 def streamOneP4Deletion(self, file):
2825 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2826 relPath = self.encodeWithUTF8(relPath)
2827 if verbose:
2828 sys.stdout.write("delete %s\n" % relPath)
2829 sys.stdout.flush()
2830 self.gitStream.write("D %s\n" % relPath)
2832 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2833 self.largeFileSystem.removeLargeFile(relPath)
2835 # handle another chunk of streaming data
2836 def streamP4FilesCb(self, marshalled):
2838 # catch p4 errors and complain
2839 err = None
2840 if "code" in marshalled:
2841 if marshalled["code"] == "error":
2842 if "data" in marshalled:
2843 err = marshalled["data"].rstrip()
2845 if not err and 'fileSize' in self.stream_file:
2846 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2847 if required_bytes > 0:
2848 err = 'Not enough space left on %s! Free at least %i MB.' % (
2849 os.getcwd(), required_bytes/1024/1024
2852 if err:
2853 f = None
2854 if self.stream_have_file_info:
2855 if "depotFile" in self.stream_file:
2856 f = self.stream_file["depotFile"]
2857 # force a failure in fast-import, else an empty
2858 # commit will be made
2859 self.gitStream.write("\n")
2860 self.gitStream.write("die-now\n")
2861 self.gitStream.close()
2862 # ignore errors, but make sure it exits first
2863 self.importProcess.wait()
2864 if f:
2865 die("Error from p4 print for %s: %s" % (f, err))
2866 else:
2867 die("Error from p4 print: %s" % err)
2869 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2870 # start of a new file - output the old one first
2871 self.streamOneP4File(self.stream_file, self.stream_contents)
2872 self.stream_file = {}
2873 self.stream_contents = []
2874 self.stream_have_file_info = False
2876 # pick up the new file information... for the
2877 # 'data' field we need to append to our array
2878 for k in marshalled.keys():
2879 if k == 'data':
2880 if 'streamContentSize' not in self.stream_file:
2881 self.stream_file['streamContentSize'] = 0
2882 self.stream_file['streamContentSize'] += len(marshalled['data'])
2883 self.stream_contents.append(marshalled['data'])
2884 else:
2885 self.stream_file[k] = marshalled[k]
2887 if (verbose and
2888 'streamContentSize' in self.stream_file and
2889 'fileSize' in self.stream_file and
2890 'depotFile' in self.stream_file):
2891 size = int(self.stream_file["fileSize"])
2892 if size > 0:
2893 progress = 100*self.stream_file['streamContentSize']/size
2894 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2895 sys.stdout.flush()
2897 self.stream_have_file_info = True
2899 # Stream directly from "p4 files" into "git fast-import"
2900 def streamP4Files(self, files):
2901 filesForCommit = []
2902 filesToRead = []
2903 filesToDelete = []
2905 for f in files:
2906 filesForCommit.append(f)
2907 if f['action'] in self.delete_actions:
2908 filesToDelete.append(f)
2909 else:
2910 filesToRead.append(f)
2912 # deleted files...
2913 for f in filesToDelete:
2914 self.streamOneP4Deletion(f)
2916 if len(filesToRead) > 0:
2917 self.stream_file = {}
2918 self.stream_contents = []
2919 self.stream_have_file_info = False
2921 # curry self argument
2922 def streamP4FilesCbSelf(entry):
2923 self.streamP4FilesCb(entry)
2925 fileArgs = []
2926 for f in filesToRead:
2927 if 'shelved_cl' in f:
2928 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2929 # the contents
2930 fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2931 else:
2932 fileArg = '%s#%s' % (f['path'], f['rev'])
2934 fileArgs.append(fileArg)
2936 p4CmdList(["-x", "-", "print"],
2937 stdin=fileArgs,
2938 cb=streamP4FilesCbSelf)
2940 # do the last chunk
2941 if self.stream_file.has_key('depotFile'):
2942 self.streamOneP4File(self.stream_file, self.stream_contents)
2944 def make_email(self, userid):
2945 if userid in self.users:
2946 return self.users[userid]
2947 else:
2948 return "%s <a@b>" % userid
2950 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2951 """ Stream a p4 tag.
2952 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2955 if verbose:
2956 print "writing tag %s for commit %s" % (labelName, commit)
2957 gitStream.write("tag %s\n" % labelName)
2958 gitStream.write("from %s\n" % commit)
2960 if labelDetails.has_key('Owner'):
2961 owner = labelDetails["Owner"]
2962 else:
2963 owner = None
2965 # Try to use the owner of the p4 label, or failing that,
2966 # the current p4 user id.
2967 if owner:
2968 email = self.make_email(owner)
2969 else:
2970 email = self.make_email(self.p4UserId())
2971 tagger = "%s %s %s" % (email, epoch, self.tz)
2973 gitStream.write("tagger %s\n" % tagger)
2975 print "labelDetails=",labelDetails
2976 if labelDetails.has_key('Description'):
2977 description = labelDetails['Description']
2978 else:
2979 description = 'Label from git p4'
2981 gitStream.write("data %d\n" % len(description))
2982 gitStream.write(description)
2983 gitStream.write("\n")
2985 def inClientSpec(self, path):
2986 if not self.clientSpecDirs:
2987 return True
2988 inClientSpec = self.clientSpecDirs.map_in_client(path)
2989 if not inClientSpec and self.verbose:
2990 print('Ignoring file outside of client spec: {0}'.format(path))
2991 return inClientSpec
2993 def hasBranchPrefix(self, path):
2994 if not self.branchPrefixes:
2995 return True
2996 hasPrefix = [p for p in self.branchPrefixes
2997 if p4PathStartsWith(path, p)]
2998 if not hasPrefix and self.verbose:
2999 print('Ignoring file outside of prefix: {0}'.format(path))
3000 return hasPrefix
3002 def commit(self, details, files, branch, parent = ""):
3003 epoch = details["time"]
3004 author = details["user"]
3005 jobs = self.extractJobsFromCommit(details)
3007 if self.verbose:
3008 print('commit into {0}'.format(branch))
3010 if self.clientSpecDirs:
3011 self.clientSpecDirs.update_client_spec_path_cache(files)
3013 files = [f for f in files
3014 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3016 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
3017 print('Ignoring revision {0} as it would produce an empty commit.'
3018 .format(details['change']))
3019 return
3021 self.gitStream.write("commit %s\n" % branch)
3022 self.gitStream.write("mark :%s\n" % details["change"])
3023 self.committedChanges.add(int(details["change"]))
3024 committer = ""
3025 if author not in self.users:
3026 self.getUserMapFromPerforceServer()
3027 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3029 self.gitStream.write("committer %s\n" % committer)
3031 self.gitStream.write("data <<EOT\n")
3032 self.gitStream.write(details["desc"])
3033 if len(jobs) > 0:
3034 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3036 if not self.suppress_meta_comment:
3037 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3038 (','.join(self.branchPrefixes), details["change"]))
3039 if len(details['options']) > 0:
3040 self.gitStream.write(": options = %s" % details['options'])
3041 self.gitStream.write("]\n")
3043 self.gitStream.write("EOT\n\n")
3045 if len(parent) > 0:
3046 if self.verbose:
3047 print "parent %s" % parent
3048 self.gitStream.write("from %s\n" % parent)
3050 self.streamP4Files(files)
3051 self.gitStream.write("\n")
3053 change = int(details["change"])
3055 if self.labels.has_key(change):
3056 label = self.labels[change]
3057 labelDetails = label[0]
3058 labelRevisions = label[1]
3059 if self.verbose:
3060 print "Change %s is labelled %s" % (change, labelDetails)
3062 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3063 for p in self.branchPrefixes])
3065 if len(files) == len(labelRevisions):
3067 cleanedFiles = {}
3068 for info in files:
3069 if info["action"] in self.delete_actions:
3070 continue
3071 cleanedFiles[info["depotFile"]] = info["rev"]
3073 if cleanedFiles == labelRevisions:
3074 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3076 else:
3077 if not self.silent:
3078 print ("Tag %s does not match with change %s: files do not match."
3079 % (labelDetails["label"], change))
3081 else:
3082 if not self.silent:
3083 print ("Tag %s does not match with change %s: file count is different."
3084 % (labelDetails["label"], change))
3086 # Build a dictionary of changelists and labels, for "detect-labels" option.
3087 def getLabels(self):
3088 self.labels = {}
3090 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3091 if len(l) > 0 and not self.silent:
3092 print "Finding files belonging to labels in %s" % `self.depotPaths`
3094 for output in l:
3095 label = output["label"]
3096 revisions = {}
3097 newestChange = 0
3098 if self.verbose:
3099 print "Querying files for label %s" % label
3100 for file in p4CmdList(["files"] +
3101 ["%s...@%s" % (p, label)
3102 for p in self.depotPaths]):
3103 revisions[file["depotFile"]] = file["rev"]
3104 change = int(file["change"])
3105 if change > newestChange:
3106 newestChange = change
3108 self.labels[newestChange] = [output, revisions]
3110 if self.verbose:
3111 print "Label changes: %s" % self.labels.keys()
3113 # Import p4 labels as git tags. A direct mapping does not
3114 # exist, so assume that if all the files are at the same revision
3115 # then we can use that, or it's something more complicated we should
3116 # just ignore.
3117 def importP4Labels(self, stream, p4Labels):
3118 if verbose:
3119 print "import p4 labels: " + ' '.join(p4Labels)
3121 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3122 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3123 if len(validLabelRegexp) == 0:
3124 validLabelRegexp = defaultLabelRegexp
3125 m = re.compile(validLabelRegexp)
3127 for name in p4Labels:
3128 commitFound = False
3130 if not m.match(name):
3131 if verbose:
3132 print "label %s does not match regexp %s" % (name,validLabelRegexp)
3133 continue
3135 if name in ignoredP4Labels:
3136 continue
3138 labelDetails = p4CmdList(['label', "-o", name])[0]
3140 # get the most recent changelist for each file in this label
3141 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3142 for p in self.depotPaths])
3144 if change.has_key('change'):
3145 # find the corresponding git commit; take the oldest commit
3146 changelist = int(change['change'])
3147 if changelist in self.committedChanges:
3148 gitCommit = ":%d" % changelist # use a fast-import mark
3149 commitFound = True
3150 else:
3151 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3152 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3153 if len(gitCommit) == 0:
3154 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
3155 else:
3156 commitFound = True
3157 gitCommit = gitCommit.strip()
3159 if commitFound:
3160 # Convert from p4 time format
3161 try:
3162 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3163 except ValueError:
3164 print "Could not convert label time %s" % labelDetails['Update']
3165 tmwhen = 1
3167 when = int(time.mktime(tmwhen))
3168 self.streamTag(stream, name, labelDetails, gitCommit, when)
3169 if verbose:
3170 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3171 else:
3172 if verbose:
3173 print "Label %s has no changelists - possibly deleted?" % name
3175 if not commitFound:
3176 # We can't import this label; don't try again as it will get very
3177 # expensive repeatedly fetching all the files for labels that will
3178 # never be imported. If the label is moved in the future, the
3179 # ignore will need to be removed manually.
3180 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3182 def guessProjectName(self):
3183 for p in self.depotPaths:
3184 if p.endswith("/"):
3185 p = p[:-1]
3186 p = p[p.strip().rfind("/") + 1:]
3187 if not p.endswith("/"):
3188 p += "/"
3189 return p
3191 def getBranchMapping(self):
3192 lostAndFoundBranches = set()
3194 user = gitConfig("git-p4.branchUser")
3195 if len(user) > 0:
3196 command = "branches -u %s" % user
3197 else:
3198 command = "branches"
3200 for info in p4CmdList(command):
3201 details = p4Cmd(["branch", "-o", info["branch"]])
3202 viewIdx = 0
3203 while details.has_key("View%s" % viewIdx):
3204 paths = details["View%s" % viewIdx].split(" ")
3205 viewIdx = viewIdx + 1
3206 # require standard //depot/foo/... //depot/bar/... mapping
3207 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3208 continue
3209 source = paths[0]
3210 destination = paths[1]
3211 ## HACK
3212 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3213 source = source[len(self.depotPaths[0]):-4]
3214 destination = destination[len(self.depotPaths[0]):-4]
3216 if destination in self.knownBranches:
3217 if not self.silent:
3218 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3219 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3220 continue
3222 self.knownBranches[destination] = source
3224 lostAndFoundBranches.discard(destination)
3226 if source not in self.knownBranches:
3227 lostAndFoundBranches.add(source)
3229 # Perforce does not strictly require branches to be defined, so we also
3230 # check git config for a branch list.
3232 # Example of branch definition in git config file:
3233 # [git-p4]
3234 # branchList=main:branchA
3235 # branchList=main:branchB
3236 # branchList=branchA:branchC
3237 configBranches = gitConfigList("git-p4.branchList")
3238 for branch in configBranches:
3239 if branch:
3240 (source, destination) = branch.split(":")
3241 self.knownBranches[destination] = source
3243 lostAndFoundBranches.discard(destination)
3245 if source not in self.knownBranches:
3246 lostAndFoundBranches.add(source)
3249 for branch in lostAndFoundBranches:
3250 self.knownBranches[branch] = branch
3252 def getBranchMappingFromGitBranches(self):
3253 branches = p4BranchesInGit(self.importIntoRemotes)
3254 for branch in branches.keys():
3255 if branch == "master":
3256 branch = "main"
3257 else:
3258 branch = branch[len(self.projectName):]
3259 self.knownBranches[branch] = branch
3261 def updateOptionDict(self, d):
3262 option_keys = {}
3263 if self.keepRepoPath:
3264 option_keys['keepRepoPath'] = 1
3266 d["options"] = ' '.join(sorted(option_keys.keys()))
3268 def readOptions(self, d):
3269 self.keepRepoPath = (d.has_key('options')
3270 and ('keepRepoPath' in d['options']))
3272 def gitRefForBranch(self, branch):
3273 if branch == "main":
3274 return self.refPrefix + "master"
3276 if len(branch) <= 0:
3277 return branch
3279 return self.refPrefix + self.projectName + branch
3281 def gitCommitByP4Change(self, ref, change):
3282 if self.verbose:
3283 print "looking in ref " + ref + " for change %s using bisect..." % change
3285 earliestCommit = ""
3286 latestCommit = parseRevision(ref)
3288 while True:
3289 if self.verbose:
3290 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3291 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3292 if len(next) == 0:
3293 if self.verbose:
3294 print "argh"
3295 return ""
3296 log = extractLogMessageFromGitCommit(next)
3297 settings = extractSettingsGitLog(log)
3298 currentChange = int(settings['change'])
3299 if self.verbose:
3300 print "current change %s" % currentChange
3302 if currentChange == change:
3303 if self.verbose:
3304 print "found %s" % next
3305 return next
3307 if currentChange < change:
3308 earliestCommit = "^%s" % next
3309 else:
3310 latestCommit = "%s" % next
3312 return ""
3314 def importNewBranch(self, branch, maxChange):
3315 # make fast-import flush all changes to disk and update the refs using the checkpoint
3316 # command so that we can try to find the branch parent in the git history
3317 self.gitStream.write("checkpoint\n\n");
3318 self.gitStream.flush();
3319 branchPrefix = self.depotPaths[0] + branch + "/"
3320 range = "@1,%s" % maxChange
3321 #print "prefix" + branchPrefix
3322 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3323 if len(changes) <= 0:
3324 return False
3325 firstChange = changes[0]
3326 #print "first change in branch: %s" % firstChange
3327 sourceBranch = self.knownBranches[branch]
3328 sourceDepotPath = self.depotPaths[0] + sourceBranch
3329 sourceRef = self.gitRefForBranch(sourceBranch)
3330 #print "source " + sourceBranch
3332 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3333 #print "branch parent: %s" % branchParentChange
3334 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3335 if len(gitParent) > 0:
3336 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3337 #print "parent git commit: %s" % gitParent
3339 self.importChanges(changes)
3340 return True
3342 def searchParent(self, parent, branch, target):
3343 parentFound = False
3344 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3345 "--no-merges", parent]):
3346 blob = blob.strip()
3347 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3348 parentFound = True
3349 if self.verbose:
3350 print "Found parent of %s in commit %s" % (branch, blob)
3351 break
3352 if parentFound:
3353 return blob
3354 else:
3355 return None
3357 def importChanges(self, changes, shelved=False, origin_revision=0):
3358 cnt = 1
3359 for change in changes:
3360 description = p4_describe(change, shelved)
3361 self.updateOptionDict(description)
3363 if not self.silent:
3364 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3365 sys.stdout.flush()
3366 cnt = cnt + 1
3368 try:
3369 if self.detectBranches:
3370 branches = self.splitFilesIntoBranches(description)
3371 for branch in branches.keys():
3372 ## HACK --hwn
3373 branchPrefix = self.depotPaths[0] + branch + "/"
3374 self.branchPrefixes = [ branchPrefix ]
3376 parent = ""
3378 filesForCommit = branches[branch]
3380 if self.verbose:
3381 print "branch is %s" % branch
3383 self.updatedBranches.add(branch)
3385 if branch not in self.createdBranches:
3386 self.createdBranches.add(branch)
3387 parent = self.knownBranches[branch]
3388 if parent == branch:
3389 parent = ""
3390 else:
3391 fullBranch = self.projectName + branch
3392 if fullBranch not in self.p4BranchesInGit:
3393 if not self.silent:
3394 print("\n Importing new branch %s" % fullBranch);
3395 if self.importNewBranch(branch, change - 1):
3396 parent = ""
3397 self.p4BranchesInGit.append(fullBranch)
3398 if not self.silent:
3399 print("\n Resuming with change %s" % change);
3401 if self.verbose:
3402 print "parent determined through known branches: %s" % parent
3404 branch = self.gitRefForBranch(branch)
3405 parent = self.gitRefForBranch(parent)
3407 if self.verbose:
3408 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3410 if len(parent) == 0 and branch in self.initialParents:
3411 parent = self.initialParents[branch]
3412 del self.initialParents[branch]
3414 blob = None
3415 if len(parent) > 0:
3416 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3417 if self.verbose:
3418 print "Creating temporary branch: " + tempBranch
3419 self.commit(description, filesForCommit, tempBranch)
3420 self.tempBranches.append(tempBranch)
3421 self.checkpoint()
3422 blob = self.searchParent(parent, branch, tempBranch)
3423 if blob:
3424 self.commit(description, filesForCommit, branch, blob)
3425 else:
3426 if self.verbose:
3427 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3428 self.commit(description, filesForCommit, branch, parent)
3429 else:
3430 files = self.extractFilesFromCommit(description, shelved, change, origin_revision)
3431 self.commit(description, files, self.branch,
3432 self.initialParent)
3433 # only needed once, to connect to the previous commit
3434 self.initialParent = ""
3435 except IOError:
3436 print self.gitError.read()
3437 sys.exit(1)
3439 def sync_origin_only(self):
3440 if self.syncWithOrigin:
3441 self.hasOrigin = originP4BranchesExist()
3442 if self.hasOrigin:
3443 if not self.silent:
3444 print 'Syncing with origin first, using "git fetch origin"'
3445 system("git fetch origin")
3447 def importHeadRevision(self, revision):
3448 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3450 details = {}
3451 details["user"] = "git perforce import user"
3452 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3453 % (' '.join(self.depotPaths), revision))
3454 details["change"] = revision
3455 newestRevision = 0
3457 fileCnt = 0
3458 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3460 for info in p4CmdList(["files"] + fileArgs):
3462 if 'code' in info and info['code'] == 'error':
3463 sys.stderr.write("p4 returned an error: %s\n"
3464 % info['data'])
3465 if info['data'].find("must refer to client") >= 0:
3466 sys.stderr.write("This particular p4 error is misleading.\n")
3467 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3468 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3469 sys.exit(1)
3470 if 'p4ExitCode' in info:
3471 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3472 sys.exit(1)
3475 change = int(info["change"])
3476 if change > newestRevision:
3477 newestRevision = change
3479 if info["action"] in self.delete_actions:
3480 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3481 #fileCnt = fileCnt + 1
3482 continue
3484 for prop in ["depotFile", "rev", "action", "type" ]:
3485 details["%s%s" % (prop, fileCnt)] = info[prop]
3487 fileCnt = fileCnt + 1
3489 details["change"] = newestRevision
3491 # Use time from top-most change so that all git p4 clones of
3492 # the same p4 repo have the same commit SHA1s.
3493 res = p4_describe(newestRevision)
3494 details["time"] = res["time"]
3496 self.updateOptionDict(details)
3497 try:
3498 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3499 except IOError:
3500 print "IO error with git fast-import. Is your git version recent enough?"
3501 print self.gitError.read()
3503 def openStreams(self):
3504 self.importProcess = subprocess.Popen(["git", "fast-import"],
3505 stdin=subprocess.PIPE,
3506 stdout=subprocess.PIPE,
3507 stderr=subprocess.PIPE);
3508 self.gitOutput = self.importProcess.stdout
3509 self.gitStream = self.importProcess.stdin
3510 self.gitError = self.importProcess.stderr
3512 def closeStreams(self):
3513 self.gitStream.close()
3514 if self.importProcess.wait() != 0:
3515 die("fast-import failed: %s" % self.gitError.read())
3516 self.gitOutput.close()
3517 self.gitError.close()
3519 def run(self, args):
3520 if self.importIntoRemotes:
3521 self.refPrefix = "refs/remotes/p4/"
3522 else:
3523 self.refPrefix = "refs/heads/p4/"
3525 self.sync_origin_only()
3527 branch_arg_given = bool(self.branch)
3528 if len(self.branch) == 0:
3529 self.branch = self.refPrefix + "master"
3530 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3531 system("git update-ref %s refs/heads/p4" % self.branch)
3532 system("git branch -D p4")
3534 # accept either the command-line option, or the configuration variable
3535 if self.useClientSpec:
3536 # will use this after clone to set the variable
3537 self.useClientSpec_from_options = True
3538 else:
3539 if gitConfigBool("git-p4.useclientspec"):
3540 self.useClientSpec = True
3541 if self.useClientSpec:
3542 self.clientSpecDirs = getClientSpec()
3544 # TODO: should always look at previous commits,
3545 # merge with previous imports, if possible.
3546 if args == []:
3547 if self.hasOrigin:
3548 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3550 # branches holds mapping from branch name to sha1
3551 branches = p4BranchesInGit(self.importIntoRemotes)
3553 # restrict to just this one, disabling detect-branches
3554 if branch_arg_given:
3555 short = self.branch.split("/")[-1]
3556 if short in branches:
3557 self.p4BranchesInGit = [ short ]
3558 else:
3559 self.p4BranchesInGit = branches.keys()
3561 if len(self.p4BranchesInGit) > 1:
3562 if not self.silent:
3563 print "Importing from/into multiple branches"
3564 self.detectBranches = True
3565 for branch in branches.keys():
3566 self.initialParents[self.refPrefix + branch] = \
3567 branches[branch]
3569 if self.verbose:
3570 print "branches: %s" % self.p4BranchesInGit
3572 p4Change = 0
3573 for branch in self.p4BranchesInGit:
3574 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3576 settings = extractSettingsGitLog(logMsg)
3578 self.readOptions(settings)
3579 if (settings.has_key('depot-paths')
3580 and settings.has_key ('change')):
3581 change = int(settings['change']) + 1
3582 p4Change = max(p4Change, change)
3584 depotPaths = sorted(settings['depot-paths'])
3585 if self.previousDepotPaths == []:
3586 self.previousDepotPaths = depotPaths
3587 else:
3588 paths = []
3589 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3590 prev_list = prev.split("/")
3591 cur_list = cur.split("/")
3592 for i in range(0, min(len(cur_list), len(prev_list))):
3593 if cur_list[i] <> prev_list[i]:
3594 i = i - 1
3595 break
3597 paths.append ("/".join(cur_list[:i + 1]))
3599 self.previousDepotPaths = paths
3601 if p4Change > 0:
3602 self.depotPaths = sorted(self.previousDepotPaths)
3603 self.changeRange = "@%s,#head" % p4Change
3604 if not self.silent and not self.detectBranches:
3605 print "Performing incremental import into %s git branch" % self.branch
3607 # accept multiple ref name abbreviations:
3608 # refs/foo/bar/branch -> use it exactly
3609 # p4/branch -> prepend refs/remotes/ or refs/heads/
3610 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3611 if not self.branch.startswith("refs/"):
3612 if self.importIntoRemotes:
3613 prepend = "refs/remotes/"
3614 else:
3615 prepend = "refs/heads/"
3616 if not self.branch.startswith("p4/"):
3617 prepend += "p4/"
3618 self.branch = prepend + self.branch
3620 if len(args) == 0 and self.depotPaths:
3621 if not self.silent:
3622 print "Depot paths: %s" % ' '.join(self.depotPaths)
3623 else:
3624 if self.depotPaths and self.depotPaths != args:
3625 print ("previous import used depot path %s and now %s was specified. "
3626 "This doesn't work!" % (' '.join (self.depotPaths),
3627 ' '.join (args)))
3628 sys.exit(1)
3630 self.depotPaths = sorted(args)
3632 revision = ""
3633 self.users = {}
3635 # Make sure no revision specifiers are used when --changesfile
3636 # is specified.
3637 bad_changesfile = False
3638 if len(self.changesFile) > 0:
3639 for p in self.depotPaths:
3640 if p.find("@") >= 0 or p.find("#") >= 0:
3641 bad_changesfile = True
3642 break
3643 if bad_changesfile:
3644 die("Option --changesfile is incompatible with revision specifiers")
3646 newPaths = []
3647 for p in self.depotPaths:
3648 if p.find("@") != -1:
3649 atIdx = p.index("@")
3650 self.changeRange = p[atIdx:]
3651 if self.changeRange == "@all":
3652 self.changeRange = ""
3653 elif ',' not in self.changeRange:
3654 revision = self.changeRange
3655 self.changeRange = ""
3656 p = p[:atIdx]
3657 elif p.find("#") != -1:
3658 hashIdx = p.index("#")
3659 revision = p[hashIdx:]
3660 p = p[:hashIdx]
3661 elif self.previousDepotPaths == []:
3662 # pay attention to changesfile, if given, else import
3663 # the entire p4 tree at the head revision
3664 if len(self.changesFile) == 0:
3665 revision = "#head"
3667 p = re.sub ("\.\.\.$", "", p)
3668 if not p.endswith("/"):
3669 p += "/"
3671 newPaths.append(p)
3673 self.depotPaths = newPaths
3675 # --detect-branches may change this for each branch
3676 self.branchPrefixes = self.depotPaths
3678 self.loadUserMapFromCache()
3679 self.labels = {}
3680 if self.detectLabels:
3681 self.getLabels();
3683 if self.detectBranches:
3684 ## FIXME - what's a P4 projectName ?
3685 self.projectName = self.guessProjectName()
3687 if self.hasOrigin:
3688 self.getBranchMappingFromGitBranches()
3689 else:
3690 self.getBranchMapping()
3691 if self.verbose:
3692 print "p4-git branches: %s" % self.p4BranchesInGit
3693 print "initial parents: %s" % self.initialParents
3694 for b in self.p4BranchesInGit:
3695 if b != "master":
3697 ## FIXME
3698 b = b[len(self.projectName):]
3699 self.createdBranches.add(b)
3701 self.openStreams()
3703 if revision:
3704 self.importHeadRevision(revision)
3705 else:
3706 changes = []
3708 if len(self.changesFile) > 0:
3709 output = open(self.changesFile).readlines()
3710 changeSet = set()
3711 for line in output:
3712 changeSet.add(int(line))
3714 for change in changeSet:
3715 changes.append(change)
3717 changes.sort()
3718 else:
3719 # catch "git p4 sync" with no new branches, in a repo that
3720 # does not have any existing p4 branches
3721 if len(args) == 0:
3722 if not self.p4BranchesInGit:
3723 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3725 # The default branch is master, unless --branch is used to
3726 # specify something else. Make sure it exists, or complain
3727 # nicely about how to use --branch.
3728 if not self.detectBranches:
3729 if not branch_exists(self.branch):
3730 if branch_arg_given:
3731 die("Error: branch %s does not exist." % self.branch)
3732 else:
3733 die("Error: no branch %s; perhaps specify one with --branch." %
3734 self.branch)
3736 if self.verbose:
3737 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3738 self.changeRange)
3739 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3741 if len(self.maxChanges) > 0:
3742 changes = changes[:min(int(self.maxChanges), len(changes))]
3744 if len(changes) == 0:
3745 if not self.silent:
3746 print "No changes to import!"
3747 else:
3748 if not self.silent and not self.detectBranches:
3749 print "Import destination: %s" % self.branch
3751 self.updatedBranches = set()
3753 if not self.detectBranches:
3754 if args:
3755 # start a new branch
3756 self.initialParent = ""
3757 else:
3758 # build on a previous revision
3759 self.initialParent = parseRevision(self.branch)
3761 self.importChanges(changes)
3763 if not self.silent:
3764 print ""
3765 if len(self.updatedBranches) > 0:
3766 sys.stdout.write("Updated branches: ")
3767 for b in self.updatedBranches:
3768 sys.stdout.write("%s " % b)
3769 sys.stdout.write("\n")
3771 if gitConfigBool("git-p4.importLabels"):
3772 self.importLabels = True
3774 if self.importLabels:
3775 p4Labels = getP4Labels(self.depotPaths)
3776 gitTags = getGitTags()
3778 missingP4Labels = p4Labels - gitTags
3779 self.importP4Labels(self.gitStream, missingP4Labels)
3781 self.closeStreams()
3783 # Cleanup temporary branches created during import
3784 if self.tempBranches != []:
3785 for branch in self.tempBranches:
3786 read_pipe("git update-ref -d %s" % branch)
3787 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3789 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3790 # a convenient shortcut refname "p4".
3791 if self.importIntoRemotes:
3792 head_ref = self.refPrefix + "HEAD"
3793 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3794 system(["git", "symbolic-ref", head_ref, self.branch])
3796 return True
3798 class P4Rebase(Command):
3799 def __init__(self):
3800 Command.__init__(self)
3801 self.options = [
3802 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3804 self.importLabels = False
3805 self.description = ("Fetches the latest revision from perforce and "
3806 + "rebases the current work (branch) against it")
3808 def run(self, args):
3809 sync = P4Sync()
3810 sync.importLabels = self.importLabels
3811 sync.run([])
3813 return self.rebase()
3815 def rebase(self):
3816 if os.system("git update-index --refresh") != 0:
3817 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3818 if len(read_pipe("git diff-index HEAD --")) > 0:
3819 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3821 [upstream, settings] = findUpstreamBranchPoint()
3822 if len(upstream) == 0:
3823 die("Cannot find upstream branchpoint for rebase")
3825 # the branchpoint may be p4/foo~3, so strip off the parent
3826 upstream = re.sub("~[0-9]+$", "", upstream)
3828 print "Rebasing the current branch onto %s" % upstream
3829 oldHead = read_pipe("git rev-parse HEAD").strip()
3830 system("git rebase %s" % upstream)
3831 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3832 return True
3834 class P4Clone(P4Sync):
3835 def __init__(self):
3836 P4Sync.__init__(self)
3837 self.description = "Creates a new git repository and imports from Perforce into it"
3838 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3839 self.options += [
3840 optparse.make_option("--destination", dest="cloneDestination",
3841 action='store', default=None,
3842 help="where to leave result of the clone"),
3843 optparse.make_option("--bare", dest="cloneBare",
3844 action="store_true", default=False),
3846 self.cloneDestination = None
3847 self.needsGit = False
3848 self.cloneBare = False
3850 def defaultDestination(self, args):
3851 ## TODO: use common prefix of args?
3852 depotPath = args[0]
3853 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3854 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3855 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3856 depotDir = re.sub(r"/$", "", depotDir)
3857 return os.path.split(depotDir)[1]
3859 def run(self, args):
3860 if len(args) < 1:
3861 return False
3863 if self.keepRepoPath and not self.cloneDestination:
3864 sys.stderr.write("Must specify destination for --keep-path\n")
3865 sys.exit(1)
3867 depotPaths = args
3869 if not self.cloneDestination and len(depotPaths) > 1:
3870 self.cloneDestination = depotPaths[-1]
3871 depotPaths = depotPaths[:-1]
3873 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3874 for p in depotPaths:
3875 if not p.startswith("//"):
3876 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3877 return False
3879 if not self.cloneDestination:
3880 self.cloneDestination = self.defaultDestination(args)
3882 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3884 if not os.path.exists(self.cloneDestination):
3885 os.makedirs(self.cloneDestination)
3886 chdir(self.cloneDestination)
3888 init_cmd = [ "git", "init" ]
3889 if self.cloneBare:
3890 init_cmd.append("--bare")
3891 retcode = subprocess.call(init_cmd)
3892 if retcode:
3893 raise CalledProcessError(retcode, init_cmd)
3895 if not P4Sync.run(self, depotPaths):
3896 return False
3898 # create a master branch and check out a work tree
3899 if gitBranchExists(self.branch):
3900 system([ "git", "branch", "master", self.branch ])
3901 if not self.cloneBare:
3902 system([ "git", "checkout", "-f" ])
3903 else:
3904 print 'Not checking out any branch, use ' \
3905 '"git checkout -q -b master <branch>"'
3907 # auto-set this variable if invoked with --use-client-spec
3908 if self.useClientSpec_from_options:
3909 system("git config --bool git-p4.useclientspec true")
3911 return True
3913 class P4Unshelve(Command):
3914 def __init__(self):
3915 Command.__init__(self)
3916 self.options = []
3917 self.origin = "HEAD"
3918 self.description = "Unshelve a P4 changelist into a git commit"
3919 self.usage = "usage: %prog [options] changelist"
3920 self.options += [
3921 optparse.make_option("--origin", dest="origin",
3922 help="Use this base revision instead of the default (%s)" % self.origin),
3924 self.verbose = False
3925 self.noCommit = False
3926 self.destbranch = "refs/remotes/p4/unshelved"
3928 def renameBranch(self, branch_name):
3929 """ Rename the existing branch to branch_name.N
3932 found = True
3933 for i in range(0,1000):
3934 backup_branch_name = "{0}.{1}".format(branch_name, i)
3935 if not gitBranchExists(backup_branch_name):
3936 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3937 gitDeleteRef(branch_name)
3938 found = True
3939 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3940 break
3942 if not found:
3943 sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3945 def findLastP4Revision(self, starting_point):
3946 """ Look back from starting_point for the first commit created by git-p4
3947 to find the P4 commit we are based on, and the depot-paths.
3950 for parent in (range(65535)):
3951 log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3952 settings = extractSettingsGitLog(log)
3953 if settings.has_key('change'):
3954 return settings
3956 sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3958 def run(self, args):
3959 if len(args) != 1:
3960 return False
3962 if not gitBranchExists(self.origin):
3963 sys.exit("origin branch {0} does not exist".format(self.origin))
3965 sync = P4Sync()
3966 changes = args
3967 sync.initialParent = self.origin
3969 # use the first change in the list to construct the branch to unshelve into
3970 change = changes[0]
3972 # if the target branch already exists, rename it
3973 branch_name = "{0}/{1}".format(self.destbranch, change)
3974 if gitBranchExists(branch_name):
3975 self.renameBranch(branch_name)
3976 sync.branch = branch_name
3978 sync.verbose = self.verbose
3979 sync.suppress_meta_comment = True
3981 settings = self.findLastP4Revision(self.origin)
3982 origin_revision = settings['change']
3983 sync.depotPaths = settings['depot-paths']
3984 sync.branchPrefixes = sync.depotPaths
3986 sync.openStreams()
3987 sync.loadUserMapFromCache()
3988 sync.silent = True
3989 sync.importChanges(changes, shelved=True, origin_revision=origin_revision)
3990 sync.closeStreams()
3992 print("unshelved changelist {0} into {1}".format(change, branch_name))
3994 return True
3996 class P4Branches(Command):
3997 def __init__(self):
3998 Command.__init__(self)
3999 self.options = [ ]
4000 self.description = ("Shows the git branches that hold imports and their "
4001 + "corresponding perforce depot paths")
4002 self.verbose = False
4004 def run(self, args):
4005 if originP4BranchesExist():
4006 createOrUpdateBranchesFromOrigin()
4008 cmdline = "git rev-parse --symbolic "
4009 cmdline += " --remotes"
4011 for line in read_pipe_lines(cmdline):
4012 line = line.strip()
4014 if not line.startswith('p4/') or line == "p4/HEAD":
4015 continue
4016 branch = line
4018 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4019 settings = extractSettingsGitLog(log)
4021 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
4022 return True
4024 class HelpFormatter(optparse.IndentedHelpFormatter):
4025 def __init__(self):
4026 optparse.IndentedHelpFormatter.__init__(self)
4028 def format_description(self, description):
4029 if description:
4030 return description + "\n"
4031 else:
4032 return ""
4034 def printUsage(commands):
4035 print "usage: %s <command> [options]" % sys.argv[0]
4036 print ""
4037 print "valid commands: %s" % ", ".join(commands)
4038 print ""
4039 print "Try %s <command> --help for command specific help." % sys.argv[0]
4040 print ""
4042 commands = {
4043 "debug" : P4Debug,
4044 "submit" : P4Submit,
4045 "commit" : P4Submit,
4046 "sync" : P4Sync,
4047 "rebase" : P4Rebase,
4048 "clone" : P4Clone,
4049 "rollback" : P4RollBack,
4050 "branches" : P4Branches,
4051 "unshelve" : P4Unshelve,
4055 def main():
4056 if len(sys.argv[1:]) == 0:
4057 printUsage(commands.keys())
4058 sys.exit(2)
4060 cmdName = sys.argv[1]
4061 try:
4062 klass = commands[cmdName]
4063 cmd = klass()
4064 except KeyError:
4065 print "unknown command %s" % cmdName
4066 print ""
4067 printUsage(commands.keys())
4068 sys.exit(2)
4070 options = cmd.options
4071 cmd.gitdir = os.environ.get("GIT_DIR", None)
4073 args = sys.argv[2:]
4075 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4076 if cmd.needsGit:
4077 options.append(optparse.make_option("--git-dir", dest="gitdir"))
4079 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4080 options,
4081 description = cmd.description,
4082 formatter = HelpFormatter())
4084 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4085 global verbose
4086 verbose = cmd.verbose
4087 if cmd.needsGit:
4088 if cmd.gitdir == None:
4089 cmd.gitdir = os.path.abspath(".git")
4090 if not isValidGitDir(cmd.gitdir):
4091 # "rev-parse --git-dir" without arguments will try $PWD/.git
4092 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4093 if os.path.exists(cmd.gitdir):
4094 cdup = read_pipe("git rev-parse --show-cdup").strip()
4095 if len(cdup) > 0:
4096 chdir(cdup);
4098 if not isValidGitDir(cmd.gitdir):
4099 if isValidGitDir(cmd.gitdir + "/.git"):
4100 cmd.gitdir += "/.git"
4101 else:
4102 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4104 # so git commands invoked from the P4 workspace will succeed
4105 os.environ["GIT_DIR"] = cmd.gitdir
4107 if not cmd.run(args):
4108 parser.print_help()
4109 sys.exit(2)
4112 if __name__ == '__main__':
4113 main()