Merge branch 'jc/drop-gen-hdrs'
[git.git] / git-p4.py
blob0b3a07cb31c4802b22dbdf8f49698973437d2dc2
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
30 # support basestring in python3
31 try:
32 unicode = unicode
33 except NameError:
34 # 'unicode' is undefined, must be Python 3
35 str = str
36 unicode = str
37 bytes = bytes
38 basestring = (str,bytes)
39 else:
40 # 'unicode' exists, must be Python 2
41 str = str
42 unicode = unicode
43 bytes = str
44 basestring = basestring
46 try:
47 from subprocess import CalledProcessError
48 except ImportError:
49 # from python2.7:subprocess.py
50 # Exception classes used by this module.
51 class CalledProcessError(Exception):
52 """This exception is raised when a process run by check_call() returns
53 a non-zero exit status. The exit status will be stored in the
54 returncode attribute."""
55 def __init__(self, returncode, cmd):
56 self.returncode = returncode
57 self.cmd = cmd
58 def __str__(self):
59 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
61 verbose = False
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
69 p4_access_checked = False
71 def p4_build_cmd(cmd):
72 """Build a suitable p4 command line.
74 This consolidates building and returning a p4 command line into one
75 location. It means that hooking into the environment, or other configuration
76 can be done more easily.
77 """
78 real_cmd = ["p4"]
80 user = gitConfig("git-p4.user")
81 if len(user) > 0:
82 real_cmd += ["-u",user]
84 password = gitConfig("git-p4.password")
85 if len(password) > 0:
86 real_cmd += ["-P", password]
88 port = gitConfig("git-p4.port")
89 if len(port) > 0:
90 real_cmd += ["-p", port]
92 host = gitConfig("git-p4.host")
93 if len(host) > 0:
94 real_cmd += ["-H", host]
96 client = gitConfig("git-p4.client")
97 if len(client) > 0:
98 real_cmd += ["-c", client]
100 retries = gitConfigInt("git-p4.retries")
101 if retries is None:
102 # Perform 3 retries by default
103 retries = 3
104 if retries > 0:
105 # Provide a way to not pass this option by setting git-p4.retries to 0
106 real_cmd += ["-r", str(retries)]
108 if isinstance(cmd,basestring):
109 real_cmd = ' '.join(real_cmd) + ' ' + cmd
110 else:
111 real_cmd += cmd
113 # now check that we can actually talk to the server
114 global p4_access_checked
115 if not p4_access_checked:
116 p4_access_checked = True # suppress access checks in p4_check_access itself
117 p4_check_access()
119 return real_cmd
121 def git_dir(path):
122 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123 This won't automatically add ".git" to a directory.
125 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126 if not d or len(d) == 0:
127 return None
128 else:
129 return d
131 def chdir(path, is_client_path=False):
132 """Do chdir to the given path, and set the PWD environment
133 variable for use by P4. It does not look at getcwd() output.
134 Since we're not using the shell, it is necessary to set the
135 PWD environment variable explicitly.
137 Normally, expand the path to force it to be absolute. This
138 addresses the use of relative path names inside P4 settings,
139 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
140 as given; it looks for .p4config using PWD.
142 If is_client_path, the path was handed to us directly by p4,
143 and may be a symbolic link. Do not call os.getcwd() in this
144 case, because it will cause p4 to think that PWD is not inside
145 the client path.
148 os.chdir(path)
149 if not is_client_path:
150 path = os.getcwd()
151 os.environ['PWD'] = path
153 def calcDiskFree():
154 """Return free space in bytes on the disk of the given dirname."""
155 if platform.system() == 'Windows':
156 free_bytes = ctypes.c_ulonglong(0)
157 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158 return free_bytes.value
159 else:
160 st = os.statvfs(os.getcwd())
161 return st.f_bavail * st.f_frsize
163 def die(msg):
164 if verbose:
165 raise Exception(msg)
166 else:
167 sys.stderr.write(msg + "\n")
168 sys.exit(1)
170 def write_pipe(c, stdin):
171 if verbose:
172 sys.stderr.write('Writing pipe: %s\n' % str(c))
174 expand = isinstance(c,basestring)
175 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176 pipe = p.stdin
177 val = pipe.write(stdin)
178 pipe.close()
179 if p.wait():
180 die('Command failed: %s' % str(c))
182 return val
184 def p4_write_pipe(c, stdin):
185 real_cmd = p4_build_cmd(c)
186 return write_pipe(real_cmd, stdin)
188 def read_pipe_full(c):
189 """ Read output from command. Returns a tuple
190 of the return status, stdout text and stderr
191 text.
193 if verbose:
194 sys.stderr.write('Reading pipe: %s\n' % str(c))
196 expand = isinstance(c,basestring)
197 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198 (out, err) = p.communicate()
199 return (p.returncode, out, err)
201 def read_pipe(c, ignore_error=False):
202 """ Read output from command. Returns the output text on
203 success. On failure, terminates execution, unless
204 ignore_error is True, when it returns an empty string.
206 (retcode, out, err) = read_pipe_full(c)
207 if retcode != 0:
208 if ignore_error:
209 out = ""
210 else:
211 die('Command failed: %s\nError: %s' % (str(c), err))
212 return out
214 def read_pipe_text(c):
215 """ Read output from a command with trailing whitespace stripped.
216 On error, returns None.
218 (retcode, out, err) = read_pipe_full(c)
219 if retcode != 0:
220 return None
221 else:
222 return out.rstrip()
224 def p4_read_pipe(c, ignore_error=False):
225 real_cmd = p4_build_cmd(c)
226 return read_pipe(real_cmd, ignore_error)
228 def read_pipe_lines(c):
229 if verbose:
230 sys.stderr.write('Reading pipe: %s\n' % str(c))
232 expand = isinstance(c, basestring)
233 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234 pipe = p.stdout
235 val = pipe.readlines()
236 if pipe.close() or p.wait():
237 die('Command failed: %s' % str(c))
239 return val
241 def p4_read_pipe_lines(c):
242 """Specifically invoke p4 on the command supplied. """
243 real_cmd = p4_build_cmd(c)
244 return read_pipe_lines(real_cmd)
246 def p4_has_command(cmd):
247 """Ask p4 for help on this command. If it returns an error, the
248 command does not exist in this version of p4."""
249 real_cmd = p4_build_cmd(["help", cmd])
250 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251 stderr=subprocess.PIPE)
252 p.communicate()
253 return p.returncode == 0
255 def p4_has_move_command():
256 """See if the move command exists, that it supports -k, and that
257 it has not been administratively disabled. The arguments
258 must be correct, but the filenames do not have to exist. Use
259 ones with wildcards so even if they exist, it will fail."""
261 if not p4_has_command("move"):
262 return False
263 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265 (out, err) = p.communicate()
266 # return code will be 1 in either case
267 if err.find("Invalid option") >= 0:
268 return False
269 if err.find("disabled") >= 0:
270 return False
271 # assume it failed because @... was invalid changelist
272 return True
274 def system(cmd, ignore_error=False):
275 expand = isinstance(cmd,basestring)
276 if verbose:
277 sys.stderr.write("executing %s\n" % str(cmd))
278 retcode = subprocess.call(cmd, shell=expand)
279 if retcode and not ignore_error:
280 raise CalledProcessError(retcode, cmd)
282 return retcode
284 def p4_system(cmd):
285 """Specifically invoke p4 as the system command. """
286 real_cmd = p4_build_cmd(cmd)
287 expand = isinstance(real_cmd, basestring)
288 retcode = subprocess.call(real_cmd, shell=expand)
289 if retcode:
290 raise CalledProcessError(retcode, real_cmd)
292 def die_bad_access(s):
293 die("failure accessing depot: {0}".format(s.rstrip()))
295 def p4_check_access(min_expiration=1):
296 """ Check if we can access Perforce - account still logged in
298 results = p4CmdList(["login", "-s"])
300 if len(results) == 0:
301 # should never get here: always get either some results, or a p4ExitCode
302 assert("could not parse response from perforce")
304 result = results[0]
306 if 'p4ExitCode' in result:
307 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308 die_bad_access("could not run p4")
310 code = result.get("code")
311 if not code:
312 # we get here if we couldn't connect and there was nothing to unmarshal
313 die_bad_access("could not connect")
315 elif code == "stat":
316 expiry = result.get("TicketExpiration")
317 if expiry:
318 expiry = int(expiry)
319 if expiry > min_expiration:
320 # ok to carry on
321 return
322 else:
323 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
325 else:
326 # account without a timeout - all ok
327 return
329 elif code == "error":
330 data = result.get("data")
331 if data:
332 die_bad_access("p4 error: {0}".format(data))
333 else:
334 die_bad_access("unknown error")
335 elif code == "info":
336 return
337 else:
338 die_bad_access("unknown error code {0}".format(code))
340 _p4_version_string = None
341 def p4_version_string():
342 """Read the version string, showing just the last line, which
343 hopefully is the interesting version bit.
345 $ p4 -V
346 Perforce - The Fast Software Configuration Management System.
347 Copyright 1995-2011 Perforce Software. All rights reserved.
348 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
350 global _p4_version_string
351 if not _p4_version_string:
352 a = p4_read_pipe_lines(["-V"])
353 _p4_version_string = a[-1].rstrip()
354 return _p4_version_string
356 def p4_integrate(src, dest):
357 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
359 def p4_sync(f, *options):
360 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
362 def p4_add(f):
363 # forcibly add file names with wildcards
364 if wildcard_present(f):
365 p4_system(["add", "-f", f])
366 else:
367 p4_system(["add", f])
369 def p4_delete(f):
370 p4_system(["delete", wildcard_encode(f)])
372 def p4_edit(f, *options):
373 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
375 def p4_revert(f):
376 p4_system(["revert", wildcard_encode(f)])
378 def p4_reopen(type, f):
379 p4_system(["reopen", "-t", type, wildcard_encode(f)])
381 def p4_reopen_in_change(changelist, files):
382 cmd = ["reopen", "-c", str(changelist)] + files
383 p4_system(cmd)
385 def p4_move(src, dest):
386 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
388 def p4_last_change():
389 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
390 return int(results[0]['change'])
392 def p4_describe(change, shelved=False):
393 """Make sure it returns a valid result by checking for
394 the presence of field "time". Return a dict of the
395 results."""
397 cmd = ["describe", "-s"]
398 if shelved:
399 cmd += ["-S"]
400 cmd += [str(change)]
402 ds = p4CmdList(cmd, skip_info=True)
403 if len(ds) != 1:
404 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
406 d = ds[0]
408 if "p4ExitCode" in d:
409 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
410 str(d)))
411 if "code" in d:
412 if d["code"] == "error":
413 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
415 if "time" not in d:
416 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
418 return d
421 # Canonicalize the p4 type and return a tuple of the
422 # base type, plus any modifiers. See "p4 help filetypes"
423 # for a list and explanation.
425 def split_p4_type(p4type):
427 p4_filetypes_historical = {
428 "ctempobj": "binary+Sw",
429 "ctext": "text+C",
430 "cxtext": "text+Cx",
431 "ktext": "text+k",
432 "kxtext": "text+kx",
433 "ltext": "text+F",
434 "tempobj": "binary+FSw",
435 "ubinary": "binary+F",
436 "uresource": "resource+F",
437 "uxbinary": "binary+Fx",
438 "xbinary": "binary+x",
439 "xltext": "text+Fx",
440 "xtempobj": "binary+Swx",
441 "xtext": "text+x",
442 "xunicode": "unicode+x",
443 "xutf16": "utf16+x",
445 if p4type in p4_filetypes_historical:
446 p4type = p4_filetypes_historical[p4type]
447 mods = ""
448 s = p4type.split("+")
449 base = s[0]
450 mods = ""
451 if len(s) > 1:
452 mods = s[1]
453 return (base, mods)
456 # return the raw p4 type of a file (text, text+ko, etc)
458 def p4_type(f):
459 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
460 return results[0]['headType']
463 # Given a type base and modifier, return a regexp matching
464 # the keywords that can be expanded in the file
466 def p4_keywords_regexp_for_type(base, type_mods):
467 if base in ("text", "unicode", "binary"):
468 kwords = None
469 if "ko" in type_mods:
470 kwords = 'Id|Header'
471 elif "k" in type_mods:
472 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
473 else:
474 return None
475 pattern = r"""
476 \$ # Starts with a dollar, followed by...
477 (%s) # one of the keywords, followed by...
478 (:[^$\n]+)? # possibly an old expansion, followed by...
479 \$ # another dollar
480 """ % kwords
481 return pattern
482 else:
483 return None
486 # Given a file, return a regexp matching the possible
487 # RCS keywords that will be expanded, or None for files
488 # with kw expansion turned off.
490 def p4_keywords_regexp_for_file(file):
491 if not os.path.exists(file):
492 return None
493 else:
494 (type_base, type_mods) = split_p4_type(p4_type(file))
495 return p4_keywords_regexp_for_type(type_base, type_mods)
497 def setP4ExecBit(file, mode):
498 # Reopens an already open file and changes the execute bit to match
499 # the execute bit setting in the passed in mode.
501 p4Type = "+x"
503 if not isModeExec(mode):
504 p4Type = getP4OpenedType(file)
505 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
506 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
507 if p4Type[-1] == "+":
508 p4Type = p4Type[0:-1]
510 p4_reopen(p4Type, file)
512 def getP4OpenedType(file):
513 # Returns the perforce file type for the given file.
515 result = p4_read_pipe(["opened", wildcard_encode(file)])
516 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
517 if match:
518 return match.group(1)
519 else:
520 die("Could not determine file type for %s (result: '%s')" % (file, result))
522 # Return the set of all p4 labels
523 def getP4Labels(depotPaths):
524 labels = set()
525 if isinstance(depotPaths,basestring):
526 depotPaths = [depotPaths]
528 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
529 label = l['label']
530 labels.add(label)
532 return labels
534 # Return the set of all git tags
535 def getGitTags():
536 gitTags = set()
537 for line in read_pipe_lines(["git", "tag"]):
538 tag = line.strip()
539 gitTags.add(tag)
540 return gitTags
542 def diffTreePattern():
543 # This is a simple generator for the diff tree regex pattern. This could be
544 # a class variable if this and parseDiffTreeEntry were a part of a class.
545 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
546 while True:
547 yield pattern
549 def parseDiffTreeEntry(entry):
550 """Parses a single diff tree entry into its component elements.
552 See git-diff-tree(1) manpage for details about the format of the diff
553 output. This method returns a dictionary with the following elements:
555 src_mode - The mode of the source file
556 dst_mode - The mode of the destination file
557 src_sha1 - The sha1 for the source file
558 dst_sha1 - The sha1 fr the destination file
559 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
560 status_score - The score for the status (applicable for 'C' and 'R'
561 statuses). This is None if there is no score.
562 src - The path for the source file.
563 dst - The path for the destination file. This is only present for
564 copy or renames. If it is not present, this is None.
566 If the pattern is not matched, None is returned."""
568 match = diffTreePattern().next().match(entry)
569 if match:
570 return {
571 'src_mode': match.group(1),
572 'dst_mode': match.group(2),
573 'src_sha1': match.group(3),
574 'dst_sha1': match.group(4),
575 'status': match.group(5),
576 'status_score': match.group(6),
577 'src': match.group(7),
578 'dst': match.group(10)
580 return None
582 def isModeExec(mode):
583 # Returns True if the given git mode represents an executable file,
584 # otherwise False.
585 return mode[-3:] == "755"
587 class P4Exception(Exception):
588 """ Base class for exceptions from the p4 client """
589 def __init__(self, exit_code):
590 self.p4ExitCode = exit_code
592 class P4ServerException(P4Exception):
593 """ Base class for exceptions where we get some kind of marshalled up result from the server """
594 def __init__(self, exit_code, p4_result):
595 super(P4ServerException, self).__init__(exit_code)
596 self.p4_result = p4_result
597 self.code = p4_result[0]['code']
598 self.data = p4_result[0]['data']
600 class P4RequestSizeException(P4ServerException):
601 """ One of the maxresults or maxscanrows errors """
602 def __init__(self, exit_code, p4_result, limit):
603 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
604 self.limit = limit
606 def isModeExecChanged(src_mode, dst_mode):
607 return isModeExec(src_mode) != isModeExec(dst_mode)
609 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
610 errors_as_exceptions=False):
612 if isinstance(cmd,basestring):
613 cmd = "-G " + cmd
614 expand = True
615 else:
616 cmd = ["-G"] + cmd
617 expand = False
619 cmd = p4_build_cmd(cmd)
620 if verbose:
621 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
623 # Use a temporary file to avoid deadlocks without
624 # subprocess.communicate(), which would put another copy
625 # of stdout into memory.
626 stdin_file = None
627 if stdin is not None:
628 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
629 if isinstance(stdin,basestring):
630 stdin_file.write(stdin)
631 else:
632 for i in stdin:
633 stdin_file.write(i + '\n')
634 stdin_file.flush()
635 stdin_file.seek(0)
637 p4 = subprocess.Popen(cmd,
638 shell=expand,
639 stdin=stdin_file,
640 stdout=subprocess.PIPE)
642 result = []
643 try:
644 while True:
645 entry = marshal.load(p4.stdout)
646 if skip_info:
647 if 'code' in entry and entry['code'] == 'info':
648 continue
649 if cb is not None:
650 cb(entry)
651 else:
652 result.append(entry)
653 except EOFError:
654 pass
655 exitCode = p4.wait()
656 if exitCode != 0:
657 if errors_as_exceptions:
658 if len(result) > 0:
659 data = result[0].get('data')
660 if data:
661 m = re.search('Too many rows scanned \(over (\d+)\)', data)
662 if not m:
663 m = re.search('Request too large \(over (\d+)\)', data)
665 if m:
666 limit = int(m.group(1))
667 raise P4RequestSizeException(exitCode, result, limit)
669 raise P4ServerException(exitCode, result)
670 else:
671 raise P4Exception(exitCode)
672 else:
673 entry = {}
674 entry["p4ExitCode"] = exitCode
675 result.append(entry)
677 return result
679 def p4Cmd(cmd):
680 list = p4CmdList(cmd)
681 result = {}
682 for entry in list:
683 result.update(entry)
684 return result;
686 def p4Where(depotPath):
687 if not depotPath.endswith("/"):
688 depotPath += "/"
689 depotPathLong = depotPath + "..."
690 outputList = p4CmdList(["where", depotPathLong])
691 output = None
692 for entry in outputList:
693 if "depotFile" in entry:
694 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
695 # The base path always ends with "/...".
696 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
697 output = entry
698 break
699 elif "data" in entry:
700 data = entry.get("data")
701 space = data.find(" ")
702 if data[:space] == depotPath:
703 output = entry
704 break
705 if output == None:
706 return ""
707 if output["code"] == "error":
708 return ""
709 clientPath = ""
710 if "path" in output:
711 clientPath = output.get("path")
712 elif "data" in output:
713 data = output.get("data")
714 lastSpace = data.rfind(" ")
715 clientPath = data[lastSpace + 1:]
717 if clientPath.endswith("..."):
718 clientPath = clientPath[:-3]
719 return clientPath
721 def currentGitBranch():
722 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
724 def isValidGitDir(path):
725 return git_dir(path) != None
727 def parseRevision(ref):
728 return read_pipe("git rev-parse %s" % ref).strip()
730 def branchExists(ref):
731 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
732 ignore_error=True)
733 return len(rev) > 0
735 def extractLogMessageFromGitCommit(commit):
736 logMessage = ""
738 ## fixme: title is first line of commit, not 1st paragraph.
739 foundTitle = False
740 for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
741 if not foundTitle:
742 if len(log) == 1:
743 foundTitle = True
744 continue
746 logMessage += log
747 return logMessage
749 def extractSettingsGitLog(log):
750 values = {}
751 for line in log.split("\n"):
752 line = line.strip()
753 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
754 if not m:
755 continue
757 assignments = m.group(1).split (':')
758 for a in assignments:
759 vals = a.split ('=')
760 key = vals[0].strip()
761 val = ('='.join (vals[1:])).strip()
762 if val.endswith ('\"') and val.startswith('"'):
763 val = val[1:-1]
765 values[key] = val
767 paths = values.get("depot-paths")
768 if not paths:
769 paths = values.get("depot-path")
770 if paths:
771 values['depot-paths'] = paths.split(',')
772 return values
774 def gitBranchExists(branch):
775 proc = subprocess.Popen(["git", "rev-parse", branch],
776 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
777 return proc.wait() == 0;
779 def gitUpdateRef(ref, newvalue):
780 subprocess.check_call(["git", "update-ref", ref, newvalue])
782 def gitDeleteRef(ref):
783 subprocess.check_call(["git", "update-ref", "-d", ref])
785 _gitConfig = {}
787 def gitConfig(key, typeSpecifier=None):
788 if key not in _gitConfig:
789 cmd = [ "git", "config" ]
790 if typeSpecifier:
791 cmd += [ typeSpecifier ]
792 cmd += [ key ]
793 s = read_pipe(cmd, ignore_error=True)
794 _gitConfig[key] = s.strip()
795 return _gitConfig[key]
797 def gitConfigBool(key):
798 """Return a bool, using git config --bool. It is True only if the
799 variable is set to true, and False if set to false or not present
800 in the config."""
802 if key not in _gitConfig:
803 _gitConfig[key] = gitConfig(key, '--bool') == "true"
804 return _gitConfig[key]
806 def gitConfigInt(key):
807 if key not in _gitConfig:
808 cmd = [ "git", "config", "--int", key ]
809 s = read_pipe(cmd, ignore_error=True)
810 v = s.strip()
811 try:
812 _gitConfig[key] = int(gitConfig(key, '--int'))
813 except ValueError:
814 _gitConfig[key] = None
815 return _gitConfig[key]
817 def gitConfigList(key):
818 if key not in _gitConfig:
819 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
820 _gitConfig[key] = s.strip().splitlines()
821 if _gitConfig[key] == ['']:
822 _gitConfig[key] = []
823 return _gitConfig[key]
825 def p4BranchesInGit(branchesAreInRemotes=True):
826 """Find all the branches whose names start with "p4/", looking
827 in remotes or heads as specified by the argument. Return
828 a dictionary of { branch: revision } for each one found.
829 The branch names are the short names, without any
830 "p4/" prefix."""
832 branches = {}
834 cmdline = "git rev-parse --symbolic "
835 if branchesAreInRemotes:
836 cmdline += "--remotes"
837 else:
838 cmdline += "--branches"
840 for line in read_pipe_lines(cmdline):
841 line = line.strip()
843 # only import to p4/
844 if not line.startswith('p4/'):
845 continue
846 # special symbolic ref to p4/master
847 if line == "p4/HEAD":
848 continue
850 # strip off p4/ prefix
851 branch = line[len("p4/"):]
853 branches[branch] = parseRevision(line)
855 return branches
857 def branch_exists(branch):
858 """Make sure that the given ref name really exists."""
860 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
861 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
862 out, _ = p.communicate()
863 if p.returncode:
864 return False
865 # expect exactly one line of output: the branch name
866 return out.rstrip() == branch
868 def findUpstreamBranchPoint(head = "HEAD"):
869 branches = p4BranchesInGit()
870 # map from depot-path to branch name
871 branchByDepotPath = {}
872 for branch in branches.keys():
873 tip = branches[branch]
874 log = extractLogMessageFromGitCommit(tip)
875 settings = extractSettingsGitLog(log)
876 if "depot-paths" in settings:
877 paths = ",".join(settings["depot-paths"])
878 branchByDepotPath[paths] = "remotes/p4/" + branch
880 settings = None
881 parent = 0
882 while parent < 65535:
883 commit = head + "~%s" % parent
884 log = extractLogMessageFromGitCommit(commit)
885 settings = extractSettingsGitLog(log)
886 if "depot-paths" in settings:
887 paths = ",".join(settings["depot-paths"])
888 if paths in branchByDepotPath:
889 return [branchByDepotPath[paths], settings]
891 parent = parent + 1
893 return ["", settings]
895 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
896 if not silent:
897 print("Creating/updating branch(es) in %s based on origin branch(es)"
898 % localRefPrefix)
900 originPrefix = "origin/p4/"
902 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
903 line = line.strip()
904 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
905 continue
907 headName = line[len(originPrefix):]
908 remoteHead = localRefPrefix + headName
909 originHead = line
911 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
912 if ('depot-paths' not in original
913 or 'change' not in original):
914 continue
916 update = False
917 if not gitBranchExists(remoteHead):
918 if verbose:
919 print("creating %s" % remoteHead)
920 update = True
921 else:
922 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
923 if 'change' in settings:
924 if settings['depot-paths'] == original['depot-paths']:
925 originP4Change = int(original['change'])
926 p4Change = int(settings['change'])
927 if originP4Change > p4Change:
928 print("%s (%s) is newer than %s (%s). "
929 "Updating p4 branch from origin."
930 % (originHead, originP4Change,
931 remoteHead, p4Change))
932 update = True
933 else:
934 print("Ignoring: %s was imported from %s while "
935 "%s was imported from %s"
936 % (originHead, ','.join(original['depot-paths']),
937 remoteHead, ','.join(settings['depot-paths'])))
939 if update:
940 system("git update-ref %s %s" % (remoteHead, originHead))
942 def originP4BranchesExist():
943 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
946 def p4ParseNumericChangeRange(parts):
947 changeStart = int(parts[0][1:])
948 if parts[1] == '#head':
949 changeEnd = p4_last_change()
950 else:
951 changeEnd = int(parts[1])
953 return (changeStart, changeEnd)
955 def chooseBlockSize(blockSize):
956 if blockSize:
957 return blockSize
958 else:
959 return defaultBlockSize
961 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
962 assert depotPaths
964 # Parse the change range into start and end. Try to find integer
965 # revision ranges as these can be broken up into blocks to avoid
966 # hitting server-side limits (maxrows, maxscanresults). But if
967 # that doesn't work, fall back to using the raw revision specifier
968 # strings, without using block mode.
970 if changeRange is None or changeRange == '':
971 changeStart = 1
972 changeEnd = p4_last_change()
973 block_size = chooseBlockSize(requestedBlockSize)
974 else:
975 parts = changeRange.split(',')
976 assert len(parts) == 2
977 try:
978 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
979 block_size = chooseBlockSize(requestedBlockSize)
980 except ValueError:
981 changeStart = parts[0][1:]
982 changeEnd = parts[1]
983 if requestedBlockSize:
984 die("cannot use --changes-block-size with non-numeric revisions")
985 block_size = None
987 changes = set()
989 # Retrieve changes a block at a time, to prevent running
990 # into a MaxResults/MaxScanRows error from the server. If
991 # we _do_ hit one of those errors, turn down the block size
993 while True:
994 cmd = ['changes']
996 if block_size:
997 end = min(changeEnd, changeStart + block_size)
998 revisionRange = "%d,%d" % (changeStart, end)
999 else:
1000 revisionRange = "%s,%s" % (changeStart, changeEnd)
1002 for p in depotPaths:
1003 cmd += ["%s...@%s" % (p, revisionRange)]
1005 # fetch the changes
1006 try:
1007 result = p4CmdList(cmd, errors_as_exceptions=True)
1008 except P4RequestSizeException as e:
1009 if not block_size:
1010 block_size = e.limit
1011 elif block_size > e.limit:
1012 block_size = e.limit
1013 else:
1014 block_size = max(2, block_size // 2)
1016 if verbose: print("block size error, retrying with block size {0}".format(block_size))
1017 continue
1018 except P4Exception as e:
1019 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1021 # Insert changes in chronological order
1022 for entry in reversed(result):
1023 if 'change' not in entry:
1024 continue
1025 changes.add(int(entry['change']))
1027 if not block_size:
1028 break
1030 if end >= changeEnd:
1031 break
1033 changeStart = end + 1
1035 changes = sorted(changes)
1036 return changes
1038 def p4PathStartsWith(path, prefix):
1039 # This method tries to remedy a potential mixed-case issue:
1041 # If UserA adds //depot/DirA/file1
1042 # and UserB adds //depot/dira/file2
1044 # we may or may not have a problem. If you have core.ignorecase=true,
1045 # we treat DirA and dira as the same directory
1046 if gitConfigBool("core.ignorecase"):
1047 return path.lower().startswith(prefix.lower())
1048 return path.startswith(prefix)
1050 def getClientSpec():
1051 """Look at the p4 client spec, create a View() object that contains
1052 all the mappings, and return it."""
1054 specList = p4CmdList("client -o")
1055 if len(specList) != 1:
1056 die('Output from "client -o" is %d lines, expecting 1' %
1057 len(specList))
1059 # dictionary of all client parameters
1060 entry = specList[0]
1062 # the //client/ name
1063 client_name = entry["Client"]
1065 # just the keys that start with "View"
1066 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1068 # hold this new View
1069 view = View(client_name)
1071 # append the lines, in order, to the view
1072 for view_num in range(len(view_keys)):
1073 k = "View%d" % view_num
1074 if k not in view_keys:
1075 die("Expected view key %s missing" % k)
1076 view.append(entry[k])
1078 return view
1080 def getClientRoot():
1081 """Grab the client directory."""
1083 output = p4CmdList("client -o")
1084 if len(output) != 1:
1085 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1087 entry = output[0]
1088 if "Root" not in entry:
1089 die('Client has no "Root"')
1091 return entry["Root"]
1094 # P4 wildcards are not allowed in filenames. P4 complains
1095 # if you simply add them, but you can force it with "-f", in
1096 # which case it translates them into %xx encoding internally.
1098 def wildcard_decode(path):
1099 # Search for and fix just these four characters. Do % last so
1100 # that fixing it does not inadvertently create new %-escapes.
1101 # Cannot have * in a filename in windows; untested as to
1102 # what p4 would do in such a case.
1103 if not platform.system() == "Windows":
1104 path = path.replace("%2A", "*")
1105 path = path.replace("%23", "#") \
1106 .replace("%40", "@") \
1107 .replace("%25", "%")
1108 return path
1110 def wildcard_encode(path):
1111 # do % first to avoid double-encoding the %s introduced here
1112 path = path.replace("%", "%25") \
1113 .replace("*", "%2A") \
1114 .replace("#", "%23") \
1115 .replace("@", "%40")
1116 return path
1118 def wildcard_present(path):
1119 m = re.search("[*#@%]", path)
1120 return m is not None
1122 class LargeFileSystem(object):
1123 """Base class for large file system support."""
1125 def __init__(self, writeToGitStream):
1126 self.largeFiles = set()
1127 self.writeToGitStream = writeToGitStream
1129 def generatePointer(self, cloneDestination, contentFile):
1130 """Return the content of a pointer file that is stored in Git instead of
1131 the actual content."""
1132 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1134 def pushFile(self, localLargeFile):
1135 """Push the actual content which is not stored in the Git repository to
1136 a server."""
1137 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1139 def hasLargeFileExtension(self, relPath):
1140 return reduce(
1141 lambda a, b: a or b,
1142 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1143 False
1146 def generateTempFile(self, contents):
1147 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1148 for d in contents:
1149 contentFile.write(d)
1150 contentFile.close()
1151 return contentFile.name
1153 def exceedsLargeFileThreshold(self, relPath, contents):
1154 if gitConfigInt('git-p4.largeFileThreshold'):
1155 contentsSize = sum(len(d) for d in contents)
1156 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1157 return True
1158 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159 contentsSize = sum(len(d) for d in contents)
1160 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1161 return False
1162 contentTempFile = self.generateTempFile(contents)
1163 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1164 with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1165 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1166 compressedContentsSize = zf.infolist()[0].compress_size
1167 os.remove(contentTempFile)
1168 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1169 return True
1170 return False
1172 def addLargeFile(self, relPath):
1173 self.largeFiles.add(relPath)
1175 def removeLargeFile(self, relPath):
1176 self.largeFiles.remove(relPath)
1178 def isLargeFile(self, relPath):
1179 return relPath in self.largeFiles
1181 def processContent(self, git_mode, relPath, contents):
1182 """Processes the content of git fast import. This method decides if a
1183 file is stored in the large file system and handles all necessary
1184 steps."""
1185 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1186 contentTempFile = self.generateTempFile(contents)
1187 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1188 if pointer_git_mode:
1189 git_mode = pointer_git_mode
1190 if localLargeFile:
1191 # Move temp file to final location in large file system
1192 largeFileDir = os.path.dirname(localLargeFile)
1193 if not os.path.isdir(largeFileDir):
1194 os.makedirs(largeFileDir)
1195 shutil.move(contentTempFile, localLargeFile)
1196 self.addLargeFile(relPath)
1197 if gitConfigBool('git-p4.largeFilePush'):
1198 self.pushFile(localLargeFile)
1199 if verbose:
1200 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1201 return (git_mode, contents)
1203 class MockLFS(LargeFileSystem):
1204 """Mock large file system for testing."""
1206 def generatePointer(self, contentFile):
1207 """The pointer content is the original content prefixed with "pointer-".
1208 The local filename of the large file storage is derived from the file content.
1210 with open(contentFile, 'r') as f:
1211 content = next(f)
1212 gitMode = '100644'
1213 pointerContents = 'pointer-' + content
1214 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1215 return (gitMode, pointerContents, localLargeFile)
1217 def pushFile(self, localLargeFile):
1218 """The remote filename of the large file storage is the same as the local
1219 one but in a different directory.
1221 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1222 if not os.path.exists(remotePath):
1223 os.makedirs(remotePath)
1224 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1226 class GitLFS(LargeFileSystem):
1227 """Git LFS as backend for the git-p4 large file system.
1228 See https://git-lfs.github.com/ for details."""
1230 def __init__(self, *args):
1231 LargeFileSystem.__init__(self, *args)
1232 self.baseGitAttributes = []
1234 def generatePointer(self, contentFile):
1235 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1236 mode and content which is stored in the Git repository instead of
1237 the actual content. Return also the new location of the actual
1238 content.
1240 if os.path.getsize(contentFile) == 0:
1241 return (None, '', None)
1243 pointerProcess = subprocess.Popen(
1244 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1245 stdout=subprocess.PIPE
1247 pointerFile = pointerProcess.stdout.read()
1248 if pointerProcess.wait():
1249 os.remove(contentFile)
1250 die('git-lfs pointer command failed. Did you install the extension?')
1252 # Git LFS removed the preamble in the output of the 'pointer' command
1253 # starting from version 1.2.0. Check for the preamble here to support
1254 # earlier versions.
1255 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1256 if pointerFile.startswith('Git LFS pointer for'):
1257 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1259 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1260 # if someone use external lfs.storage ( not in local repo git )
1261 lfs_path = gitConfig('lfs.storage')
1262 if not lfs_path:
1263 lfs_path = 'lfs'
1264 if not os.path.isabs(lfs_path):
1265 lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
1266 localLargeFile = os.path.join(
1267 lfs_path,
1268 'objects', oid[:2], oid[2:4],
1269 oid,
1271 # LFS Spec states that pointer files should not have the executable bit set.
1272 gitMode = '100644'
1273 return (gitMode, pointerFile, localLargeFile)
1275 def pushFile(self, localLargeFile):
1276 uploadProcess = subprocess.Popen(
1277 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1279 if uploadProcess.wait():
1280 die('git-lfs push command failed. Did you define a remote?')
1282 def generateGitAttributes(self):
1283 return (
1284 self.baseGitAttributes +
1286 '\n',
1287 '#\n',
1288 '# Git LFS (see https://git-lfs.github.com/)\n',
1289 '#\n',
1291 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1292 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1294 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1295 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1299 def addLargeFile(self, relPath):
1300 LargeFileSystem.addLargeFile(self, relPath)
1301 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1303 def removeLargeFile(self, relPath):
1304 LargeFileSystem.removeLargeFile(self, relPath)
1305 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1307 def processContent(self, git_mode, relPath, contents):
1308 if relPath == '.gitattributes':
1309 self.baseGitAttributes = contents
1310 return (git_mode, self.generateGitAttributes())
1311 else:
1312 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1314 class Command:
1315 delete_actions = ( "delete", "move/delete", "purge" )
1316 add_actions = ( "add", "branch", "move/add" )
1318 def __init__(self):
1319 self.usage = "usage: %prog [options]"
1320 self.needsGit = True
1321 self.verbose = False
1323 # This is required for the "append" update_shelve action
1324 def ensure_value(self, attr, value):
1325 if not hasattr(self, attr) or getattr(self, attr) is None:
1326 setattr(self, attr, value)
1327 return getattr(self, attr)
1329 class P4UserMap:
1330 def __init__(self):
1331 self.userMapFromPerforceServer = False
1332 self.myP4UserId = None
1334 def p4UserId(self):
1335 if self.myP4UserId:
1336 return self.myP4UserId
1338 results = p4CmdList("user -o")
1339 for r in results:
1340 if 'User' in r:
1341 self.myP4UserId = r['User']
1342 return r['User']
1343 die("Could not find your p4 user id")
1345 def p4UserIsMe(self, p4User):
1346 # return True if the given p4 user is actually me
1347 me = self.p4UserId()
1348 if not p4User or p4User != me:
1349 return False
1350 else:
1351 return True
1353 def getUserCacheFilename(self):
1354 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1355 return home + "/.gitp4-usercache.txt"
1357 def getUserMapFromPerforceServer(self):
1358 if self.userMapFromPerforceServer:
1359 return
1360 self.users = {}
1361 self.emails = {}
1363 for output in p4CmdList("users"):
1364 if "User" not in output:
1365 continue
1366 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1367 self.emails[output["Email"]] = output["User"]
1369 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1370 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1371 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1372 if mapUser and len(mapUser[0]) == 3:
1373 user = mapUser[0][0]
1374 fullname = mapUser[0][1]
1375 email = mapUser[0][2]
1376 self.users[user] = fullname + " <" + email + ">"
1377 self.emails[email] = user
1379 s = ''
1380 for (key, val) in self.users.items():
1381 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1383 open(self.getUserCacheFilename(), "wb").write(s)
1384 self.userMapFromPerforceServer = True
1386 def loadUserMapFromCache(self):
1387 self.users = {}
1388 self.userMapFromPerforceServer = False
1389 try:
1390 cache = open(self.getUserCacheFilename(), "rb")
1391 lines = cache.readlines()
1392 cache.close()
1393 for line in lines:
1394 entry = line.strip().split("\t")
1395 self.users[entry[0]] = entry[1]
1396 except IOError:
1397 self.getUserMapFromPerforceServer()
1399 class P4Debug(Command):
1400 def __init__(self):
1401 Command.__init__(self)
1402 self.options = []
1403 self.description = "A tool to debug the output of p4 -G."
1404 self.needsGit = False
1406 def run(self, args):
1407 j = 0
1408 for output in p4CmdList(args):
1409 print('Element: %d' % j)
1410 j += 1
1411 print(output)
1412 return True
1414 class P4RollBack(Command):
1415 def __init__(self):
1416 Command.__init__(self)
1417 self.options = [
1418 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1420 self.description = "A tool to debug the multi-branch import. Don't use :)"
1421 self.rollbackLocalBranches = False
1423 def run(self, args):
1424 if len(args) != 1:
1425 return False
1426 maxChange = int(args[0])
1428 if "p4ExitCode" in p4Cmd("changes -m 1"):
1429 die("Problems executing p4");
1431 if self.rollbackLocalBranches:
1432 refPrefix = "refs/heads/"
1433 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1434 else:
1435 refPrefix = "refs/remotes/"
1436 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1438 for line in lines:
1439 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1440 line = line.strip()
1441 ref = refPrefix + line
1442 log = extractLogMessageFromGitCommit(ref)
1443 settings = extractSettingsGitLog(log)
1445 depotPaths = settings['depot-paths']
1446 change = settings['change']
1448 changed = False
1450 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1451 for p in depotPaths]))) == 0:
1452 print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1453 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1454 continue
1456 while change and int(change) > maxChange:
1457 changed = True
1458 if self.verbose:
1459 print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1460 system("git update-ref %s \"%s^\"" % (ref, ref))
1461 log = extractLogMessageFromGitCommit(ref)
1462 settings = extractSettingsGitLog(log)
1465 depotPaths = settings['depot-paths']
1466 change = settings['change']
1468 if changed:
1469 print("%s rewound to %s" % (ref, change))
1471 return True
1473 class P4Submit(Command, P4UserMap):
1475 conflict_behavior_choices = ("ask", "skip", "quit")
1477 def __init__(self):
1478 Command.__init__(self)
1479 P4UserMap.__init__(self)
1480 self.options = [
1481 optparse.make_option("--origin", dest="origin"),
1482 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1483 # preserve the user, requires relevant p4 permissions
1484 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1485 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1486 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1487 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1488 optparse.make_option("--conflict", dest="conflict_behavior",
1489 choices=self.conflict_behavior_choices),
1490 optparse.make_option("--branch", dest="branch"),
1491 optparse.make_option("--shelve", dest="shelve", action="store_true",
1492 help="Shelve instead of submit. Shelved files are reverted, "
1493 "restoring the workspace to the state before the shelve"),
1494 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1495 metavar="CHANGELIST",
1496 help="update an existing shelved changelist, implies --shelve, "
1497 "repeat in-order for multiple shelved changelists"),
1498 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1499 help="submit only the specified commit(s), one commit or xxx..xxx"),
1500 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1501 help="Disable rebase after submit is completed. Can be useful if you "
1502 "work from a local git branch that is not master"),
1503 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1504 help="Skip Perforce sync of p4/master after submit or shelve"),
1506 self.description = """Submit changes from git to the perforce depot.\n
1507 The `p4-pre-submit` hook is executed if it exists and is executable.
1508 The hook takes no parameters and nothing from standard input. Exiting with
1509 non-zero status from this script prevents `git-p4 submit` from launching.
1511 One usage scenario is to run unit tests in the hook."""
1513 self.usage += " [name of git branch to submit into perforce depot]"
1514 self.origin = ""
1515 self.detectRenames = False
1516 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1517 self.dry_run = False
1518 self.shelve = False
1519 self.update_shelve = list()
1520 self.commit = ""
1521 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1522 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1523 self.prepare_p4_only = False
1524 self.conflict_behavior = None
1525 self.isWindows = (platform.system() == "Windows")
1526 self.exportLabels = False
1527 self.p4HasMoveCommand = p4_has_move_command()
1528 self.branch = None
1530 if gitConfig('git-p4.largeFileSystem'):
1531 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1533 def check(self):
1534 if len(p4CmdList("opened ...")) > 0:
1535 die("You have files opened with perforce! Close them before starting the sync.")
1537 def separate_jobs_from_description(self, message):
1538 """Extract and return a possible Jobs field in the commit
1539 message. It goes into a separate section in the p4 change
1540 specification.
1542 A jobs line starts with "Jobs:" and looks like a new field
1543 in a form. Values are white-space separated on the same
1544 line or on following lines that start with a tab.
1546 This does not parse and extract the full git commit message
1547 like a p4 form. It just sees the Jobs: line as a marker
1548 to pass everything from then on directly into the p4 form,
1549 but outside the description section.
1551 Return a tuple (stripped log message, jobs string)."""
1553 m = re.search(r'^Jobs:', message, re.MULTILINE)
1554 if m is None:
1555 return (message, None)
1557 jobtext = message[m.start():]
1558 stripped_message = message[:m.start()].rstrip()
1559 return (stripped_message, jobtext)
1561 def prepareLogMessage(self, template, message, jobs):
1562 """Edits the template returned from "p4 change -o" to insert
1563 the message in the Description field, and the jobs text in
1564 the Jobs field."""
1565 result = ""
1567 inDescriptionSection = False
1569 for line in template.split("\n"):
1570 if line.startswith("#"):
1571 result += line + "\n"
1572 continue
1574 if inDescriptionSection:
1575 if line.startswith("Files:") or line.startswith("Jobs:"):
1576 inDescriptionSection = False
1577 # insert Jobs section
1578 if jobs:
1579 result += jobs + "\n"
1580 else:
1581 continue
1582 else:
1583 if line.startswith("Description:"):
1584 inDescriptionSection = True
1585 line += "\n"
1586 for messageLine in message.split("\n"):
1587 line += "\t" + messageLine + "\n"
1589 result += line + "\n"
1591 return result
1593 def patchRCSKeywords(self, file, pattern):
1594 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1595 (handle, outFileName) = tempfile.mkstemp(dir='.')
1596 try:
1597 outFile = os.fdopen(handle, "w+")
1598 inFile = open(file, "r")
1599 regexp = re.compile(pattern, re.VERBOSE)
1600 for line in inFile.readlines():
1601 line = regexp.sub(r'$\1$', line)
1602 outFile.write(line)
1603 inFile.close()
1604 outFile.close()
1605 # Forcibly overwrite the original file
1606 os.unlink(file)
1607 shutil.move(outFileName, file)
1608 except:
1609 # cleanup our temporary file
1610 os.unlink(outFileName)
1611 print("Failed to strip RCS keywords in %s" % file)
1612 raise
1614 print("Patched up RCS keywords in %s" % file)
1616 def p4UserForCommit(self,id):
1617 # Return the tuple (perforce user,git email) for a given git commit id
1618 self.getUserMapFromPerforceServer()
1619 gitEmail = read_pipe(["git", "log", "--max-count=1",
1620 "--format=%ae", id])
1621 gitEmail = gitEmail.strip()
1622 if gitEmail not in self.emails:
1623 return (None,gitEmail)
1624 else:
1625 return (self.emails[gitEmail],gitEmail)
1627 def checkValidP4Users(self,commits):
1628 # check if any git authors cannot be mapped to p4 users
1629 for id in commits:
1630 (user,email) = self.p4UserForCommit(id)
1631 if not user:
1632 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1633 if gitConfigBool("git-p4.allowMissingP4Users"):
1634 print("%s" % msg)
1635 else:
1636 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1638 def lastP4Changelist(self):
1639 # Get back the last changelist number submitted in this client spec. This
1640 # then gets used to patch up the username in the change. If the same
1641 # client spec is being used by multiple processes then this might go
1642 # wrong.
1643 results = p4CmdList("client -o") # find the current client
1644 client = None
1645 for r in results:
1646 if 'Client' in r:
1647 client = r['Client']
1648 break
1649 if not client:
1650 die("could not get client spec")
1651 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1652 for r in results:
1653 if 'change' in r:
1654 return r['change']
1655 die("Could not get changelist number for last submit - cannot patch up user details")
1657 def modifyChangelistUser(self, changelist, newUser):
1658 # fixup the user field of a changelist after it has been submitted.
1659 changes = p4CmdList("change -o %s" % changelist)
1660 if len(changes) != 1:
1661 die("Bad output from p4 change modifying %s to user %s" %
1662 (changelist, newUser))
1664 c = changes[0]
1665 if c['User'] == newUser: return # nothing to do
1666 c['User'] = newUser
1667 input = marshal.dumps(c)
1669 result = p4CmdList("change -f -i", stdin=input)
1670 for r in result:
1671 if 'code' in r:
1672 if r['code'] == 'error':
1673 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1674 if 'data' in r:
1675 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1676 return
1677 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1679 def canChangeChangelists(self):
1680 # check to see if we have p4 admin or super-user permissions, either of
1681 # which are required to modify changelists.
1682 results = p4CmdList(["protects", self.depotPath])
1683 for r in results:
1684 if 'perm' in r:
1685 if r['perm'] == 'admin':
1686 return 1
1687 if r['perm'] == 'super':
1688 return 1
1689 return 0
1691 def prepareSubmitTemplate(self, changelist=None):
1692 """Run "p4 change -o" to grab a change specification template.
1693 This does not use "p4 -G", as it is nice to keep the submission
1694 template in original order, since a human might edit it.
1696 Remove lines in the Files section that show changes to files
1697 outside the depot path we're committing into."""
1699 [upstream, settings] = findUpstreamBranchPoint()
1701 template = """\
1702 # A Perforce Change Specification.
1704 # Change: The change number. 'new' on a new changelist.
1705 # Date: The date this specification was last modified.
1706 # Client: The client on which the changelist was created. Read-only.
1707 # User: The user who created the changelist.
1708 # Status: Either 'pending' or 'submitted'. Read-only.
1709 # Type: Either 'public' or 'restricted'. Default is 'public'.
1710 # Description: Comments about the changelist. Required.
1711 # Jobs: What opened jobs are to be closed by this changelist.
1712 # You may delete jobs from this list. (New changelists only.)
1713 # Files: What opened files from the default changelist are to be added
1714 # to this changelist. You may delete files from this list.
1715 # (New changelists only.)
1717 files_list = []
1718 inFilesSection = False
1719 change_entry = None
1720 args = ['change', '-o']
1721 if changelist:
1722 args.append(str(changelist))
1723 for entry in p4CmdList(args):
1724 if 'code' not in entry:
1725 continue
1726 if entry['code'] == 'stat':
1727 change_entry = entry
1728 break
1729 if not change_entry:
1730 die('Failed to decode output of p4 change -o')
1731 for key, value in change_entry.iteritems():
1732 if key.startswith('File'):
1733 if 'depot-paths' in settings:
1734 if not [p for p in settings['depot-paths']
1735 if p4PathStartsWith(value, p)]:
1736 continue
1737 else:
1738 if not p4PathStartsWith(value, self.depotPath):
1739 continue
1740 files_list.append(value)
1741 continue
1742 # Output in the order expected by prepareLogMessage
1743 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1744 if key not in change_entry:
1745 continue
1746 template += '\n'
1747 template += key + ':'
1748 if key == 'Description':
1749 template += '\n'
1750 for field_line in change_entry[key].splitlines():
1751 template += '\t'+field_line+'\n'
1752 if len(files_list) > 0:
1753 template += '\n'
1754 template += 'Files:\n'
1755 for path in files_list:
1756 template += '\t'+path+'\n'
1757 return template
1759 def edit_template(self, template_file):
1760 """Invoke the editor to let the user change the submission
1761 message. Return true if okay to continue with the submit."""
1763 # if configured to skip the editing part, just submit
1764 if gitConfigBool("git-p4.skipSubmitEdit"):
1765 return True
1767 # look at the modification time, to check later if the user saved
1768 # the file
1769 mtime = os.stat(template_file).st_mtime
1771 # invoke the editor
1772 if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1773 editor = os.environ.get("P4EDITOR")
1774 else:
1775 editor = read_pipe("git var GIT_EDITOR").strip()
1776 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1778 # If the file was not saved, prompt to see if this patch should
1779 # be skipped. But skip this verification step if configured so.
1780 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1781 return True
1783 # modification time updated means user saved the file
1784 if os.stat(template_file).st_mtime > mtime:
1785 return True
1787 while True:
1788 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1789 if response == 'y':
1790 return True
1791 if response == 'n':
1792 return False
1794 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1795 # diff
1796 if "P4DIFF" in os.environ:
1797 del(os.environ["P4DIFF"])
1798 diff = ""
1799 for editedFile in editedFiles:
1800 diff += p4_read_pipe(['diff', '-du',
1801 wildcard_encode(editedFile)])
1803 # new file diff
1804 newdiff = ""
1805 for newFile in filesToAdd:
1806 newdiff += "==== new file ====\n"
1807 newdiff += "--- /dev/null\n"
1808 newdiff += "+++ %s\n" % newFile
1810 is_link = os.path.islink(newFile)
1811 expect_link = newFile in symlinks
1813 if is_link and expect_link:
1814 newdiff += "+%s\n" % os.readlink(newFile)
1815 else:
1816 f = open(newFile, "r")
1817 for line in f.readlines():
1818 newdiff += "+" + line
1819 f.close()
1821 return (diff + newdiff).replace('\r\n', '\n')
1823 def applyCommit(self, id):
1824 """Apply one commit, return True if it succeeded."""
1826 print("Applying", read_pipe(["git", "show", "-s",
1827 "--format=format:%h %s", id]))
1829 (p4User, gitEmail) = self.p4UserForCommit(id)
1831 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1832 filesToAdd = set()
1833 filesToChangeType = set()
1834 filesToDelete = set()
1835 editedFiles = set()
1836 pureRenameCopy = set()
1837 symlinks = set()
1838 filesToChangeExecBit = {}
1839 all_files = list()
1841 for line in diff:
1842 diff = parseDiffTreeEntry(line)
1843 modifier = diff['status']
1844 path = diff['src']
1845 all_files.append(path)
1847 if modifier == "M":
1848 p4_edit(path)
1849 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1850 filesToChangeExecBit[path] = diff['dst_mode']
1851 editedFiles.add(path)
1852 elif modifier == "A":
1853 filesToAdd.add(path)
1854 filesToChangeExecBit[path] = diff['dst_mode']
1855 if path in filesToDelete:
1856 filesToDelete.remove(path)
1858 dst_mode = int(diff['dst_mode'], 8)
1859 if dst_mode == 0o120000:
1860 symlinks.add(path)
1862 elif modifier == "D":
1863 filesToDelete.add(path)
1864 if path in filesToAdd:
1865 filesToAdd.remove(path)
1866 elif modifier == "C":
1867 src, dest = diff['src'], diff['dst']
1868 all_files.append(dest)
1869 p4_integrate(src, dest)
1870 pureRenameCopy.add(dest)
1871 if diff['src_sha1'] != diff['dst_sha1']:
1872 p4_edit(dest)
1873 pureRenameCopy.discard(dest)
1874 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1875 p4_edit(dest)
1876 pureRenameCopy.discard(dest)
1877 filesToChangeExecBit[dest] = diff['dst_mode']
1878 if self.isWindows:
1879 # turn off read-only attribute
1880 os.chmod(dest, stat.S_IWRITE)
1881 os.unlink(dest)
1882 editedFiles.add(dest)
1883 elif modifier == "R":
1884 src, dest = diff['src'], diff['dst']
1885 all_files.append(dest)
1886 if self.p4HasMoveCommand:
1887 p4_edit(src) # src must be open before move
1888 p4_move(src, dest) # opens for (move/delete, move/add)
1889 else:
1890 p4_integrate(src, dest)
1891 if diff['src_sha1'] != diff['dst_sha1']:
1892 p4_edit(dest)
1893 else:
1894 pureRenameCopy.add(dest)
1895 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1896 if not self.p4HasMoveCommand:
1897 p4_edit(dest) # with move: already open, writable
1898 filesToChangeExecBit[dest] = diff['dst_mode']
1899 if not self.p4HasMoveCommand:
1900 if self.isWindows:
1901 os.chmod(dest, stat.S_IWRITE)
1902 os.unlink(dest)
1903 filesToDelete.add(src)
1904 editedFiles.add(dest)
1905 elif modifier == "T":
1906 filesToChangeType.add(path)
1907 else:
1908 die("unknown modifier %s for %s" % (modifier, path))
1910 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1911 patchcmd = diffcmd + " | git apply "
1912 tryPatchCmd = patchcmd + "--check -"
1913 applyPatchCmd = patchcmd + "--check --apply -"
1914 patch_succeeded = True
1916 if os.system(tryPatchCmd) != 0:
1917 fixed_rcs_keywords = False
1918 patch_succeeded = False
1919 print("Unfortunately applying the change failed!")
1921 # Patch failed, maybe it's just RCS keyword woes. Look through
1922 # the patch to see if that's possible.
1923 if gitConfigBool("git-p4.attemptRCSCleanup"):
1924 file = None
1925 pattern = None
1926 kwfiles = {}
1927 for file in editedFiles | filesToDelete:
1928 # did this file's delta contain RCS keywords?
1929 pattern = p4_keywords_regexp_for_file(file)
1931 if pattern:
1932 # this file is a possibility...look for RCS keywords.
1933 regexp = re.compile(pattern, re.VERBOSE)
1934 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1935 if regexp.search(line):
1936 if verbose:
1937 print("got keyword match on %s in %s in %s" % (pattern, line, file))
1938 kwfiles[file] = pattern
1939 break
1941 for file in kwfiles:
1942 if verbose:
1943 print("zapping %s with %s" % (line,pattern))
1944 # File is being deleted, so not open in p4. Must
1945 # disable the read-only bit on windows.
1946 if self.isWindows and file not in editedFiles:
1947 os.chmod(file, stat.S_IWRITE)
1948 self.patchRCSKeywords(file, kwfiles[file])
1949 fixed_rcs_keywords = True
1951 if fixed_rcs_keywords:
1952 print("Retrying the patch with RCS keywords cleaned up")
1953 if os.system(tryPatchCmd) == 0:
1954 patch_succeeded = True
1956 if not patch_succeeded:
1957 for f in editedFiles:
1958 p4_revert(f)
1959 return False
1962 # Apply the patch for real, and do add/delete/+x handling.
1964 system(applyPatchCmd)
1966 for f in filesToChangeType:
1967 p4_edit(f, "-t", "auto")
1968 for f in filesToAdd:
1969 p4_add(f)
1970 for f in filesToDelete:
1971 p4_revert(f)
1972 p4_delete(f)
1974 # Set/clear executable bits
1975 for f in filesToChangeExecBit.keys():
1976 mode = filesToChangeExecBit[f]
1977 setP4ExecBit(f, mode)
1979 update_shelve = 0
1980 if len(self.update_shelve) > 0:
1981 update_shelve = self.update_shelve.pop(0)
1982 p4_reopen_in_change(update_shelve, all_files)
1985 # Build p4 change description, starting with the contents
1986 # of the git commit message.
1988 logMessage = extractLogMessageFromGitCommit(id)
1989 logMessage = logMessage.strip()
1990 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1992 template = self.prepareSubmitTemplate(update_shelve)
1993 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1995 if self.preserveUser:
1996 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1998 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1999 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2000 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2001 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2003 separatorLine = "######## everything below this line is just the diff #######\n"
2004 if not self.prepare_p4_only:
2005 submitTemplate += separatorLine
2006 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2008 (handle, fileName) = tempfile.mkstemp()
2009 tmpFile = os.fdopen(handle, "w+b")
2010 if self.isWindows:
2011 submitTemplate = submitTemplate.replace("\n", "\r\n")
2012 tmpFile.write(submitTemplate)
2013 tmpFile.close()
2015 if self.prepare_p4_only:
2017 # Leave the p4 tree prepared, and the submit template around
2018 # and let the user decide what to do next
2020 print()
2021 print("P4 workspace prepared for submission.")
2022 print("To submit or revert, go to client workspace")
2023 print(" " + self.clientPath)
2024 print()
2025 print("To submit, use \"p4 submit\" to write a new description,")
2026 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2027 " \"git p4\"." % fileName)
2028 print("You can delete the file \"%s\" when finished." % fileName)
2030 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2031 print("To preserve change ownership by user %s, you must\n" \
2032 "do \"p4 change -f <change>\" after submitting and\n" \
2033 "edit the User field.")
2034 if pureRenameCopy:
2035 print("After submitting, renamed files must be re-synced.")
2036 print("Invoke \"p4 sync -f\" on each of these files:")
2037 for f in pureRenameCopy:
2038 print(" " + f)
2040 print()
2041 print("To revert the changes, use \"p4 revert ...\", and delete")
2042 print("the submit template file \"%s\"" % fileName)
2043 if filesToAdd:
2044 print("Since the commit adds new files, they must be deleted:")
2045 for f in filesToAdd:
2046 print(" " + f)
2047 print()
2048 return True
2051 # Let the user edit the change description, then submit it.
2053 submitted = False
2055 try:
2056 if self.edit_template(fileName):
2057 # read the edited message and submit
2058 tmpFile = open(fileName, "rb")
2059 message = tmpFile.read()
2060 tmpFile.close()
2061 if self.isWindows:
2062 message = message.replace("\r\n", "\n")
2063 submitTemplate = message[:message.index(separatorLine)]
2065 if update_shelve:
2066 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2067 elif self.shelve:
2068 p4_write_pipe(['shelve', '-i'], submitTemplate)
2069 else:
2070 p4_write_pipe(['submit', '-i'], submitTemplate)
2071 # The rename/copy happened by applying a patch that created a
2072 # new file. This leaves it writable, which confuses p4.
2073 for f in pureRenameCopy:
2074 p4_sync(f, "-f")
2076 if self.preserveUser:
2077 if p4User:
2078 # Get last changelist number. Cannot easily get it from
2079 # the submit command output as the output is
2080 # unmarshalled.
2081 changelist = self.lastP4Changelist()
2082 self.modifyChangelistUser(changelist, p4User)
2084 submitted = True
2086 finally:
2087 # skip this patch
2088 if not submitted or self.shelve:
2089 if self.shelve:
2090 print ("Reverting shelved files.")
2091 else:
2092 print ("Submission cancelled, undoing p4 changes.")
2093 for f in editedFiles | filesToDelete:
2094 p4_revert(f)
2095 for f in filesToAdd:
2096 p4_revert(f)
2097 os.remove(f)
2099 os.remove(fileName)
2100 return submitted
2102 # Export git tags as p4 labels. Create a p4 label and then tag
2103 # with that.
2104 def exportGitTags(self, gitTags):
2105 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2106 if len(validLabelRegexp) == 0:
2107 validLabelRegexp = defaultLabelRegexp
2108 m = re.compile(validLabelRegexp)
2110 for name in gitTags:
2112 if not m.match(name):
2113 if verbose:
2114 print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2115 continue
2117 # Get the p4 commit this corresponds to
2118 logMessage = extractLogMessageFromGitCommit(name)
2119 values = extractSettingsGitLog(logMessage)
2121 if 'change' not in values:
2122 # a tag pointing to something not sent to p4; ignore
2123 if verbose:
2124 print("git tag %s does not give a p4 commit" % name)
2125 continue
2126 else:
2127 changelist = values['change']
2129 # Get the tag details.
2130 inHeader = True
2131 isAnnotated = False
2132 body = []
2133 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2134 l = l.strip()
2135 if inHeader:
2136 if re.match(r'tag\s+', l):
2137 isAnnotated = True
2138 elif re.match(r'\s*$', l):
2139 inHeader = False
2140 continue
2141 else:
2142 body.append(l)
2144 if not isAnnotated:
2145 body = ["lightweight tag imported by git p4\n"]
2147 # Create the label - use the same view as the client spec we are using
2148 clientSpec = getClientSpec()
2150 labelTemplate = "Label: %s\n" % name
2151 labelTemplate += "Description:\n"
2152 for b in body:
2153 labelTemplate += "\t" + b + "\n"
2154 labelTemplate += "View:\n"
2155 for depot_side in clientSpec.mappings:
2156 labelTemplate += "\t%s\n" % depot_side
2158 if self.dry_run:
2159 print("Would create p4 label %s for tag" % name)
2160 elif self.prepare_p4_only:
2161 print("Not creating p4 label %s for tag due to option" \
2162 " --prepare-p4-only" % name)
2163 else:
2164 p4_write_pipe(["label", "-i"], labelTemplate)
2166 # Use the label
2167 p4_system(["tag", "-l", name] +
2168 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2170 if verbose:
2171 print("created p4 label for tag %s" % name)
2173 def run(self, args):
2174 if len(args) == 0:
2175 self.master = currentGitBranch()
2176 elif len(args) == 1:
2177 self.master = args[0]
2178 if not branchExists(self.master):
2179 die("Branch %s does not exist" % self.master)
2180 else:
2181 return False
2183 for i in self.update_shelve:
2184 if i <= 0:
2185 sys.exit("invalid changelist %d" % i)
2187 if self.master:
2188 allowSubmit = gitConfig("git-p4.allowSubmit")
2189 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2190 die("%s is not in git-p4.allowSubmit" % self.master)
2192 [upstream, settings] = findUpstreamBranchPoint()
2193 self.depotPath = settings['depot-paths'][0]
2194 if len(self.origin) == 0:
2195 self.origin = upstream
2197 if len(self.update_shelve) > 0:
2198 self.shelve = True
2200 if self.preserveUser:
2201 if not self.canChangeChangelists():
2202 die("Cannot preserve user names without p4 super-user or admin permissions")
2204 # if not set from the command line, try the config file
2205 if self.conflict_behavior is None:
2206 val = gitConfig("git-p4.conflict")
2207 if val:
2208 if val not in self.conflict_behavior_choices:
2209 die("Invalid value '%s' for config git-p4.conflict" % val)
2210 else:
2211 val = "ask"
2212 self.conflict_behavior = val
2214 if self.verbose:
2215 print("Origin branch is " + self.origin)
2217 if len(self.depotPath) == 0:
2218 print("Internal error: cannot locate perforce depot path from existing branches")
2219 sys.exit(128)
2221 self.useClientSpec = False
2222 if gitConfigBool("git-p4.useclientspec"):
2223 self.useClientSpec = True
2224 if self.useClientSpec:
2225 self.clientSpecDirs = getClientSpec()
2227 # Check for the existence of P4 branches
2228 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2230 if self.useClientSpec and not branchesDetected:
2231 # all files are relative to the client spec
2232 self.clientPath = getClientRoot()
2233 else:
2234 self.clientPath = p4Where(self.depotPath)
2236 if self.clientPath == "":
2237 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2239 print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2240 self.oldWorkingDirectory = os.getcwd()
2242 # ensure the clientPath exists
2243 new_client_dir = False
2244 if not os.path.exists(self.clientPath):
2245 new_client_dir = True
2246 os.makedirs(self.clientPath)
2248 chdir(self.clientPath, is_client_path=True)
2249 if self.dry_run:
2250 print("Would synchronize p4 checkout in %s" % self.clientPath)
2251 else:
2252 print("Synchronizing p4 checkout...")
2253 if new_client_dir:
2254 # old one was destroyed, and maybe nobody told p4
2255 p4_sync("...", "-f")
2256 else:
2257 p4_sync("...")
2258 self.check()
2260 commits = []
2261 if self.master:
2262 committish = self.master
2263 else:
2264 committish = 'HEAD'
2266 if self.commit != "":
2267 if self.commit.find("..") != -1:
2268 limits_ish = self.commit.split("..")
2269 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2270 commits.append(line.strip())
2271 commits.reverse()
2272 else:
2273 commits.append(self.commit)
2274 else:
2275 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2276 commits.append(line.strip())
2277 commits.reverse()
2279 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2280 self.checkAuthorship = False
2281 else:
2282 self.checkAuthorship = True
2284 if self.preserveUser:
2285 self.checkValidP4Users(commits)
2288 # Build up a set of options to be passed to diff when
2289 # submitting each commit to p4.
2291 if self.detectRenames:
2292 # command-line -M arg
2293 self.diffOpts = "-M"
2294 else:
2295 # If not explicitly set check the config variable
2296 detectRenames = gitConfig("git-p4.detectRenames")
2298 if detectRenames.lower() == "false" or detectRenames == "":
2299 self.diffOpts = ""
2300 elif detectRenames.lower() == "true":
2301 self.diffOpts = "-M"
2302 else:
2303 self.diffOpts = "-M%s" % detectRenames
2305 # no command-line arg for -C or --find-copies-harder, just
2306 # config variables
2307 detectCopies = gitConfig("git-p4.detectCopies")
2308 if detectCopies.lower() == "false" or detectCopies == "":
2309 pass
2310 elif detectCopies.lower() == "true":
2311 self.diffOpts += " -C"
2312 else:
2313 self.diffOpts += " -C%s" % detectCopies
2315 if gitConfigBool("git-p4.detectCopiesHarder"):
2316 self.diffOpts += " --find-copies-harder"
2318 num_shelves = len(self.update_shelve)
2319 if num_shelves > 0 and num_shelves != len(commits):
2320 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2321 (len(commits), num_shelves))
2323 hooks_path = gitConfig("core.hooksPath")
2324 if len(hooks_path) <= 0:
2325 hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2327 hook_file = os.path.join(hooks_path, "p4-pre-submit")
2328 if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2329 sys.exit(1)
2332 # Apply the commits, one at a time. On failure, ask if should
2333 # continue to try the rest of the patches, or quit.
2335 if self.dry_run:
2336 print("Would apply")
2337 applied = []
2338 last = len(commits) - 1
2339 for i, commit in enumerate(commits):
2340 if self.dry_run:
2341 print(" ", read_pipe(["git", "show", "-s",
2342 "--format=format:%h %s", commit]))
2343 ok = True
2344 else:
2345 ok = self.applyCommit(commit)
2346 if ok:
2347 applied.append(commit)
2348 else:
2349 if self.prepare_p4_only and i < last:
2350 print("Processing only the first commit due to option" \
2351 " --prepare-p4-only")
2352 break
2353 if i < last:
2354 quit = False
2355 while True:
2356 # prompt for what to do, or use the option/variable
2357 if self.conflict_behavior == "ask":
2358 print("What do you want to do?")
2359 response = raw_input("[s]kip this commit but apply"
2360 " the rest, or [q]uit? ")
2361 if not response:
2362 continue
2363 elif self.conflict_behavior == "skip":
2364 response = "s"
2365 elif self.conflict_behavior == "quit":
2366 response = "q"
2367 else:
2368 die("Unknown conflict_behavior '%s'" %
2369 self.conflict_behavior)
2371 if response[0] == "s":
2372 print("Skipping this commit, but applying the rest")
2373 break
2374 if response[0] == "q":
2375 print("Quitting")
2376 quit = True
2377 break
2378 if quit:
2379 break
2381 chdir(self.oldWorkingDirectory)
2382 shelved_applied = "shelved" if self.shelve else "applied"
2383 if self.dry_run:
2384 pass
2385 elif self.prepare_p4_only:
2386 pass
2387 elif len(commits) == len(applied):
2388 print("All commits {0}!".format(shelved_applied))
2390 sync = P4Sync()
2391 if self.branch:
2392 sync.branch = self.branch
2393 if self.disable_p4sync:
2394 sync.sync_origin_only()
2395 else:
2396 sync.run([])
2398 if not self.disable_rebase:
2399 rebase = P4Rebase()
2400 rebase.rebase()
2402 else:
2403 if len(applied) == 0:
2404 print("No commits {0}.".format(shelved_applied))
2405 else:
2406 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2407 for c in commits:
2408 if c in applied:
2409 star = "*"
2410 else:
2411 star = " "
2412 print(star, read_pipe(["git", "show", "-s",
2413 "--format=format:%h %s", c]))
2414 print("You will have to do 'git p4 sync' and rebase.")
2416 if gitConfigBool("git-p4.exportLabels"):
2417 self.exportLabels = True
2419 if self.exportLabels:
2420 p4Labels = getP4Labels(self.depotPath)
2421 gitTags = getGitTags()
2423 missingGitTags = gitTags - p4Labels
2424 self.exportGitTags(missingGitTags)
2426 # exit with error unless everything applied perfectly
2427 if len(commits) != len(applied):
2428 sys.exit(1)
2430 return True
2432 class View(object):
2433 """Represent a p4 view ("p4 help views"), and map files in a
2434 repo according to the view."""
2436 def __init__(self, client_name):
2437 self.mappings = []
2438 self.client_prefix = "//%s/" % client_name
2439 # cache results of "p4 where" to lookup client file locations
2440 self.client_spec_path_cache = {}
2442 def append(self, view_line):
2443 """Parse a view line, splitting it into depot and client
2444 sides. Append to self.mappings, preserving order. This
2445 is only needed for tag creation."""
2447 # Split the view line into exactly two words. P4 enforces
2448 # structure on these lines that simplifies this quite a bit.
2450 # Either or both words may be double-quoted.
2451 # Single quotes do not matter.
2452 # Double-quote marks cannot occur inside the words.
2453 # A + or - prefix is also inside the quotes.
2454 # There are no quotes unless they contain a space.
2455 # The line is already white-space stripped.
2456 # The two words are separated by a single space.
2458 if view_line[0] == '"':
2459 # First word is double quoted. Find its end.
2460 close_quote_index = view_line.find('"', 1)
2461 if close_quote_index <= 0:
2462 die("No first-word closing quote found: %s" % view_line)
2463 depot_side = view_line[1:close_quote_index]
2464 # skip closing quote and space
2465 rhs_index = close_quote_index + 1 + 1
2466 else:
2467 space_index = view_line.find(" ")
2468 if space_index <= 0:
2469 die("No word-splitting space found: %s" % view_line)
2470 depot_side = view_line[0:space_index]
2471 rhs_index = space_index + 1
2473 # prefix + means overlay on previous mapping
2474 if depot_side.startswith("+"):
2475 depot_side = depot_side[1:]
2477 # prefix - means exclude this path, leave out of mappings
2478 exclude = False
2479 if depot_side.startswith("-"):
2480 exclude = True
2481 depot_side = depot_side[1:]
2483 if not exclude:
2484 self.mappings.append(depot_side)
2486 def convert_client_path(self, clientFile):
2487 # chop off //client/ part to make it relative
2488 if not clientFile.startswith(self.client_prefix):
2489 die("No prefix '%s' on clientFile '%s'" %
2490 (self.client_prefix, clientFile))
2491 return clientFile[len(self.client_prefix):]
2493 def update_client_spec_path_cache(self, files):
2494 """ Caching file paths by "p4 where" batch query """
2496 # List depot file paths exclude that already cached
2497 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2499 if len(fileArgs) == 0:
2500 return # All files in cache
2502 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2503 for res in where_result:
2504 if "code" in res and res["code"] == "error":
2505 # assume error is "... file(s) not in client view"
2506 continue
2507 if "clientFile" not in res:
2508 die("No clientFile in 'p4 where' output")
2509 if "unmap" in res:
2510 # it will list all of them, but only one not unmap-ped
2511 continue
2512 if gitConfigBool("core.ignorecase"):
2513 res['depotFile'] = res['depotFile'].lower()
2514 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2516 # not found files or unmap files set to ""
2517 for depotFile in fileArgs:
2518 if gitConfigBool("core.ignorecase"):
2519 depotFile = depotFile.lower()
2520 if depotFile not in self.client_spec_path_cache:
2521 self.client_spec_path_cache[depotFile] = ""
2523 def map_in_client(self, depot_path):
2524 """Return the relative location in the client where this
2525 depot file should live. Returns "" if the file should
2526 not be mapped in the client."""
2528 if gitConfigBool("core.ignorecase"):
2529 depot_path = depot_path.lower()
2531 if depot_path in self.client_spec_path_cache:
2532 return self.client_spec_path_cache[depot_path]
2534 die( "Error: %s is not found in client spec path" % depot_path )
2535 return ""
2537 def cloneExcludeCallback(option, opt_str, value, parser):
2538 # prepend "/" because the first "/" was consumed as part of the option itself.
2539 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2540 parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2542 class P4Sync(Command, P4UserMap):
2544 def __init__(self):
2545 Command.__init__(self)
2546 P4UserMap.__init__(self)
2547 self.options = [
2548 optparse.make_option("--branch", dest="branch"),
2549 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2550 optparse.make_option("--changesfile", dest="changesFile"),
2551 optparse.make_option("--silent", dest="silent", action="store_true"),
2552 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2553 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2554 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2555 help="Import into refs/heads/ , not refs/remotes"),
2556 optparse.make_option("--max-changes", dest="maxChanges",
2557 help="Maximum number of changes to import"),
2558 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2559 help="Internal block size to use when iteratively calling p4 changes"),
2560 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2561 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2562 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2563 help="Only sync files that are included in the Perforce Client Spec"),
2564 optparse.make_option("-/", dest="cloneExclude",
2565 action="callback", callback=cloneExcludeCallback, type="string",
2566 help="exclude depot path"),
2568 self.description = """Imports from Perforce into a git repository.\n
2569 example:
2570 //depot/my/project/ -- to import the current head
2571 //depot/my/project/@all -- to import everything
2572 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2574 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2576 self.usage += " //depot/path[@revRange]"
2577 self.silent = False
2578 self.createdBranches = set()
2579 self.committedChanges = set()
2580 self.branch = ""
2581 self.detectBranches = False
2582 self.detectLabels = False
2583 self.importLabels = False
2584 self.changesFile = ""
2585 self.syncWithOrigin = True
2586 self.importIntoRemotes = True
2587 self.maxChanges = ""
2588 self.changes_block_size = None
2589 self.keepRepoPath = False
2590 self.depotPaths = None
2591 self.p4BranchesInGit = []
2592 self.cloneExclude = []
2593 self.useClientSpec = False
2594 self.useClientSpec_from_options = False
2595 self.clientSpecDirs = None
2596 self.tempBranches = []
2597 self.tempBranchLocation = "refs/git-p4-tmp"
2598 self.largeFileSystem = None
2599 self.suppress_meta_comment = False
2601 if gitConfig('git-p4.largeFileSystem'):
2602 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2603 self.largeFileSystem = largeFileSystemConstructor(
2604 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2607 if gitConfig("git-p4.syncFromOrigin") == "false":
2608 self.syncWithOrigin = False
2610 self.depotPaths = []
2611 self.changeRange = ""
2612 self.previousDepotPaths = []
2613 self.hasOrigin = False
2615 # map from branch depot path to parent branch
2616 self.knownBranches = {}
2617 self.initialParents = {}
2619 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2620 self.labels = {}
2622 # Force a checkpoint in fast-import and wait for it to finish
2623 def checkpoint(self):
2624 self.gitStream.write("checkpoint\n\n")
2625 self.gitStream.write("progress checkpoint\n\n")
2626 out = self.gitOutput.readline()
2627 if self.verbose:
2628 print("checkpoint finished: " + out)
2630 def isPathWanted(self, path):
2631 for p in self.cloneExclude:
2632 if p.endswith("/"):
2633 if p4PathStartsWith(path, p):
2634 return False
2635 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2636 elif path.lower() == p.lower():
2637 return False
2638 for p in self.depotPaths:
2639 if p4PathStartsWith(path, p):
2640 return True
2641 return False
2643 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2644 files = []
2645 fnum = 0
2646 while "depotFile%s" % fnum in commit:
2647 path = commit["depotFile%s" % fnum]
2648 found = self.isPathWanted(path)
2649 if not found:
2650 fnum = fnum + 1
2651 continue
2653 file = {}
2654 file["path"] = path
2655 file["rev"] = commit["rev%s" % fnum]
2656 file["action"] = commit["action%s" % fnum]
2657 file["type"] = commit["type%s" % fnum]
2658 if shelved:
2659 file["shelved_cl"] = int(shelved_cl)
2660 files.append(file)
2661 fnum = fnum + 1
2662 return files
2664 def extractJobsFromCommit(self, commit):
2665 jobs = []
2666 jnum = 0
2667 while "job%s" % jnum in commit:
2668 job = commit["job%s" % jnum]
2669 jobs.append(job)
2670 jnum = jnum + 1
2671 return jobs
2673 def stripRepoPath(self, path, prefixes):
2674 """When streaming files, this is called to map a p4 depot path
2675 to where it should go in git. The prefixes are either
2676 self.depotPaths, or self.branchPrefixes in the case of
2677 branch detection."""
2679 if self.useClientSpec:
2680 # branch detection moves files up a level (the branch name)
2681 # from what client spec interpretation gives
2682 path = self.clientSpecDirs.map_in_client(path)
2683 if self.detectBranches:
2684 for b in self.knownBranches:
2685 if p4PathStartsWith(path, b + "/"):
2686 path = path[len(b)+1:]
2688 elif self.keepRepoPath:
2689 # Preserve everything in relative path name except leading
2690 # //depot/; just look at first prefix as they all should
2691 # be in the same depot.
2692 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2693 if p4PathStartsWith(path, depot):
2694 path = path[len(depot):]
2696 else:
2697 for p in prefixes:
2698 if p4PathStartsWith(path, p):
2699 path = path[len(p):]
2700 break
2702 path = wildcard_decode(path)
2703 return path
2705 def splitFilesIntoBranches(self, commit):
2706 """Look at each depotFile in the commit to figure out to what
2707 branch it belongs."""
2709 if self.clientSpecDirs:
2710 files = self.extractFilesFromCommit(commit)
2711 self.clientSpecDirs.update_client_spec_path_cache(files)
2713 branches = {}
2714 fnum = 0
2715 while "depotFile%s" % fnum in commit:
2716 path = commit["depotFile%s" % fnum]
2717 found = self.isPathWanted(path)
2718 if not found:
2719 fnum = fnum + 1
2720 continue
2722 file = {}
2723 file["path"] = path
2724 file["rev"] = commit["rev%s" % fnum]
2725 file["action"] = commit["action%s" % fnum]
2726 file["type"] = commit["type%s" % fnum]
2727 fnum = fnum + 1
2729 # start with the full relative path where this file would
2730 # go in a p4 client
2731 if self.useClientSpec:
2732 relPath = self.clientSpecDirs.map_in_client(path)
2733 else:
2734 relPath = self.stripRepoPath(path, self.depotPaths)
2736 for branch in self.knownBranches.keys():
2737 # add a trailing slash so that a commit into qt/4.2foo
2738 # doesn't end up in qt/4.2, e.g.
2739 if p4PathStartsWith(relPath, branch + "/"):
2740 if branch not in branches:
2741 branches[branch] = []
2742 branches[branch].append(file)
2743 break
2745 return branches
2747 def writeToGitStream(self, gitMode, relPath, contents):
2748 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2749 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2750 for d in contents:
2751 self.gitStream.write(d)
2752 self.gitStream.write('\n')
2754 def encodeWithUTF8(self, path):
2755 try:
2756 path.decode('ascii')
2757 except:
2758 encoding = 'utf8'
2759 if gitConfig('git-p4.pathEncoding'):
2760 encoding = gitConfig('git-p4.pathEncoding')
2761 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2762 if self.verbose:
2763 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2764 return path
2766 # output one file from the P4 stream
2767 # - helper for streamP4Files
2769 def streamOneP4File(self, file, contents):
2770 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2771 relPath = self.encodeWithUTF8(relPath)
2772 if verbose:
2773 if 'fileSize' in self.stream_file:
2774 size = int(self.stream_file['fileSize'])
2775 else:
2776 size = 0 # deleted files don't get a fileSize apparently
2777 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2778 sys.stdout.flush()
2780 (type_base, type_mods) = split_p4_type(file["type"])
2782 git_mode = "100644"
2783 if "x" in type_mods:
2784 git_mode = "100755"
2785 if type_base == "symlink":
2786 git_mode = "120000"
2787 # p4 print on a symlink sometimes contains "target\n";
2788 # if it does, remove the newline
2789 data = ''.join(contents)
2790 if not data:
2791 # Some version of p4 allowed creating a symlink that pointed
2792 # to nothing. This causes p4 errors when checking out such
2793 # a change, and errors here too. Work around it by ignoring
2794 # the bad symlink; hopefully a future change fixes it.
2795 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2796 return
2797 elif data[-1] == '\n':
2798 contents = [data[:-1]]
2799 else:
2800 contents = [data]
2802 if type_base == "utf16":
2803 # p4 delivers different text in the python output to -G
2804 # than it does when using "print -o", or normal p4 client
2805 # operations. utf16 is converted to ascii or utf8, perhaps.
2806 # But ascii text saved as -t utf16 is completely mangled.
2807 # Invoke print -o to get the real contents.
2809 # On windows, the newlines will always be mangled by print, so put
2810 # them back too. This is not needed to the cygwin windows version,
2811 # just the native "NT" type.
2813 try:
2814 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2815 except Exception as e:
2816 if 'Translation of file content failed' in str(e):
2817 type_base = 'binary'
2818 else:
2819 raise e
2820 else:
2821 if p4_version_string().find('/NT') >= 0:
2822 text = text.replace('\r\n', '\n')
2823 contents = [ text ]
2825 if type_base == "apple":
2826 # Apple filetype files will be streamed as a concatenation of
2827 # its appledouble header and the contents. This is useless
2828 # on both macs and non-macs. If using "print -q -o xx", it
2829 # will create "xx" with the data, and "%xx" with the header.
2830 # This is also not very useful.
2832 # Ideally, someday, this script can learn how to generate
2833 # appledouble files directly and import those to git, but
2834 # non-mac machines can never find a use for apple filetype.
2835 print("\nIgnoring apple filetype file %s" % file['depotFile'])
2836 return
2838 # Note that we do not try to de-mangle keywords on utf16 files,
2839 # even though in theory somebody may want that.
2840 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2841 if pattern:
2842 regexp = re.compile(pattern, re.VERBOSE)
2843 text = ''.join(contents)
2844 text = regexp.sub(r'$\1$', text)
2845 contents = [ text ]
2847 if self.largeFileSystem:
2848 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2850 self.writeToGitStream(git_mode, relPath, contents)
2852 def streamOneP4Deletion(self, file):
2853 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2854 relPath = self.encodeWithUTF8(relPath)
2855 if verbose:
2856 sys.stdout.write("delete %s\n" % relPath)
2857 sys.stdout.flush()
2858 self.gitStream.write("D %s\n" % relPath)
2860 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2861 self.largeFileSystem.removeLargeFile(relPath)
2863 # handle another chunk of streaming data
2864 def streamP4FilesCb(self, marshalled):
2866 # catch p4 errors and complain
2867 err = None
2868 if "code" in marshalled:
2869 if marshalled["code"] == "error":
2870 if "data" in marshalled:
2871 err = marshalled["data"].rstrip()
2873 if not err and 'fileSize' in self.stream_file:
2874 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2875 if required_bytes > 0:
2876 err = 'Not enough space left on %s! Free at least %i MB.' % (
2877 os.getcwd(), required_bytes/1024/1024
2880 if err:
2881 f = None
2882 if self.stream_have_file_info:
2883 if "depotFile" in self.stream_file:
2884 f = self.stream_file["depotFile"]
2885 # force a failure in fast-import, else an empty
2886 # commit will be made
2887 self.gitStream.write("\n")
2888 self.gitStream.write("die-now\n")
2889 self.gitStream.close()
2890 # ignore errors, but make sure it exits first
2891 self.importProcess.wait()
2892 if f:
2893 die("Error from p4 print for %s: %s" % (f, err))
2894 else:
2895 die("Error from p4 print: %s" % err)
2897 if 'depotFile' in marshalled and self.stream_have_file_info:
2898 # start of a new file - output the old one first
2899 self.streamOneP4File(self.stream_file, self.stream_contents)
2900 self.stream_file = {}
2901 self.stream_contents = []
2902 self.stream_have_file_info = False
2904 # pick up the new file information... for the
2905 # 'data' field we need to append to our array
2906 for k in marshalled.keys():
2907 if k == 'data':
2908 if 'streamContentSize' not in self.stream_file:
2909 self.stream_file['streamContentSize'] = 0
2910 self.stream_file['streamContentSize'] += len(marshalled['data'])
2911 self.stream_contents.append(marshalled['data'])
2912 else:
2913 self.stream_file[k] = marshalled[k]
2915 if (verbose and
2916 'streamContentSize' in self.stream_file and
2917 'fileSize' in self.stream_file and
2918 'depotFile' in self.stream_file):
2919 size = int(self.stream_file["fileSize"])
2920 if size > 0:
2921 progress = 100*self.stream_file['streamContentSize']/size
2922 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2923 sys.stdout.flush()
2925 self.stream_have_file_info = True
2927 # Stream directly from "p4 files" into "git fast-import"
2928 def streamP4Files(self, files):
2929 filesForCommit = []
2930 filesToRead = []
2931 filesToDelete = []
2933 for f in files:
2934 filesForCommit.append(f)
2935 if f['action'] in self.delete_actions:
2936 filesToDelete.append(f)
2937 else:
2938 filesToRead.append(f)
2940 # deleted files...
2941 for f in filesToDelete:
2942 self.streamOneP4Deletion(f)
2944 if len(filesToRead) > 0:
2945 self.stream_file = {}
2946 self.stream_contents = []
2947 self.stream_have_file_info = False
2949 # curry self argument
2950 def streamP4FilesCbSelf(entry):
2951 self.streamP4FilesCb(entry)
2953 fileArgs = []
2954 for f in filesToRead:
2955 if 'shelved_cl' in f:
2956 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2957 # the contents
2958 fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2959 else:
2960 fileArg = '%s#%s' % (f['path'], f['rev'])
2962 fileArgs.append(fileArg)
2964 p4CmdList(["-x", "-", "print"],
2965 stdin=fileArgs,
2966 cb=streamP4FilesCbSelf)
2968 # do the last chunk
2969 if 'depotFile' in self.stream_file:
2970 self.streamOneP4File(self.stream_file, self.stream_contents)
2972 def make_email(self, userid):
2973 if userid in self.users:
2974 return self.users[userid]
2975 else:
2976 return "%s <a@b>" % userid
2978 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2979 """ Stream a p4 tag.
2980 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2983 if verbose:
2984 print("writing tag %s for commit %s" % (labelName, commit))
2985 gitStream.write("tag %s\n" % labelName)
2986 gitStream.write("from %s\n" % commit)
2988 if 'Owner' in labelDetails:
2989 owner = labelDetails["Owner"]
2990 else:
2991 owner = None
2993 # Try to use the owner of the p4 label, or failing that,
2994 # the current p4 user id.
2995 if owner:
2996 email = self.make_email(owner)
2997 else:
2998 email = self.make_email(self.p4UserId())
2999 tagger = "%s %s %s" % (email, epoch, self.tz)
3001 gitStream.write("tagger %s\n" % tagger)
3003 print("labelDetails=",labelDetails)
3004 if 'Description' in labelDetails:
3005 description = labelDetails['Description']
3006 else:
3007 description = 'Label from git p4'
3009 gitStream.write("data %d\n" % len(description))
3010 gitStream.write(description)
3011 gitStream.write("\n")
3013 def inClientSpec(self, path):
3014 if not self.clientSpecDirs:
3015 return True
3016 inClientSpec = self.clientSpecDirs.map_in_client(path)
3017 if not inClientSpec and self.verbose:
3018 print('Ignoring file outside of client spec: {0}'.format(path))
3019 return inClientSpec
3021 def hasBranchPrefix(self, path):
3022 if not self.branchPrefixes:
3023 return True
3024 hasPrefix = [p for p in self.branchPrefixes
3025 if p4PathStartsWith(path, p)]
3026 if not hasPrefix and self.verbose:
3027 print('Ignoring file outside of prefix: {0}'.format(path))
3028 return hasPrefix
3030 def commit(self, details, files, branch, parent = "", allow_empty=False):
3031 epoch = details["time"]
3032 author = details["user"]
3033 jobs = self.extractJobsFromCommit(details)
3035 if self.verbose:
3036 print('commit into {0}'.format(branch))
3038 if self.clientSpecDirs:
3039 self.clientSpecDirs.update_client_spec_path_cache(files)
3041 files = [f for f in files
3042 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3044 if gitConfigBool('git-p4.keepEmptyCommits'):
3045 allow_empty = True
3047 if not files and not allow_empty:
3048 print('Ignoring revision {0} as it would produce an empty commit.'
3049 .format(details['change']))
3050 return
3052 self.gitStream.write("commit %s\n" % branch)
3053 self.gitStream.write("mark :%s\n" % details["change"])
3054 self.committedChanges.add(int(details["change"]))
3055 committer = ""
3056 if author not in self.users:
3057 self.getUserMapFromPerforceServer()
3058 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3060 self.gitStream.write("committer %s\n" % committer)
3062 self.gitStream.write("data <<EOT\n")
3063 self.gitStream.write(details["desc"])
3064 if len(jobs) > 0:
3065 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3067 if not self.suppress_meta_comment:
3068 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3069 (','.join(self.branchPrefixes), details["change"]))
3070 if len(details['options']) > 0:
3071 self.gitStream.write(": options = %s" % details['options'])
3072 self.gitStream.write("]\n")
3074 self.gitStream.write("EOT\n\n")
3076 if len(parent) > 0:
3077 if self.verbose:
3078 print("parent %s" % parent)
3079 self.gitStream.write("from %s\n" % parent)
3081 self.streamP4Files(files)
3082 self.gitStream.write("\n")
3084 change = int(details["change"])
3086 if change in self.labels:
3087 label = self.labels[change]
3088 labelDetails = label[0]
3089 labelRevisions = label[1]
3090 if self.verbose:
3091 print("Change %s is labelled %s" % (change, labelDetails))
3093 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3094 for p in self.branchPrefixes])
3096 if len(files) == len(labelRevisions):
3098 cleanedFiles = {}
3099 for info in files:
3100 if info["action"] in self.delete_actions:
3101 continue
3102 cleanedFiles[info["depotFile"]] = info["rev"]
3104 if cleanedFiles == labelRevisions:
3105 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3107 else:
3108 if not self.silent:
3109 print("Tag %s does not match with change %s: files do not match."
3110 % (labelDetails["label"], change))
3112 else:
3113 if not self.silent:
3114 print("Tag %s does not match with change %s: file count is different."
3115 % (labelDetails["label"], change))
3117 # Build a dictionary of changelists and labels, for "detect-labels" option.
3118 def getLabels(self):
3119 self.labels = {}
3121 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3122 if len(l) > 0 and not self.silent:
3123 print("Finding files belonging to labels in %s" % self.depotPaths)
3125 for output in l:
3126 label = output["label"]
3127 revisions = {}
3128 newestChange = 0
3129 if self.verbose:
3130 print("Querying files for label %s" % label)
3131 for file in p4CmdList(["files"] +
3132 ["%s...@%s" % (p, label)
3133 for p in self.depotPaths]):
3134 revisions[file["depotFile"]] = file["rev"]
3135 change = int(file["change"])
3136 if change > newestChange:
3137 newestChange = change
3139 self.labels[newestChange] = [output, revisions]
3141 if self.verbose:
3142 print("Label changes: %s" % self.labels.keys())
3144 # Import p4 labels as git tags. A direct mapping does not
3145 # exist, so assume that if all the files are at the same revision
3146 # then we can use that, or it's something more complicated we should
3147 # just ignore.
3148 def importP4Labels(self, stream, p4Labels):
3149 if verbose:
3150 print("import p4 labels: " + ' '.join(p4Labels))
3152 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3153 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3154 if len(validLabelRegexp) == 0:
3155 validLabelRegexp = defaultLabelRegexp
3156 m = re.compile(validLabelRegexp)
3158 for name in p4Labels:
3159 commitFound = False
3161 if not m.match(name):
3162 if verbose:
3163 print("label %s does not match regexp %s" % (name,validLabelRegexp))
3164 continue
3166 if name in ignoredP4Labels:
3167 continue
3169 labelDetails = p4CmdList(['label', "-o", name])[0]
3171 # get the most recent changelist for each file in this label
3172 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3173 for p in self.depotPaths])
3175 if 'change' in change:
3176 # find the corresponding git commit; take the oldest commit
3177 changelist = int(change['change'])
3178 if changelist in self.committedChanges:
3179 gitCommit = ":%d" % changelist # use a fast-import mark
3180 commitFound = True
3181 else:
3182 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3183 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3184 if len(gitCommit) == 0:
3185 print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3186 else:
3187 commitFound = True
3188 gitCommit = gitCommit.strip()
3190 if commitFound:
3191 # Convert from p4 time format
3192 try:
3193 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3194 except ValueError:
3195 print("Could not convert label time %s" % labelDetails['Update'])
3196 tmwhen = 1
3198 when = int(time.mktime(tmwhen))
3199 self.streamTag(stream, name, labelDetails, gitCommit, when)
3200 if verbose:
3201 print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3202 else:
3203 if verbose:
3204 print("Label %s has no changelists - possibly deleted?" % name)
3206 if not commitFound:
3207 # We can't import this label; don't try again as it will get very
3208 # expensive repeatedly fetching all the files for labels that will
3209 # never be imported. If the label is moved in the future, the
3210 # ignore will need to be removed manually.
3211 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3213 def guessProjectName(self):
3214 for p in self.depotPaths:
3215 if p.endswith("/"):
3216 p = p[:-1]
3217 p = p[p.strip().rfind("/") + 1:]
3218 if not p.endswith("/"):
3219 p += "/"
3220 return p
3222 def getBranchMapping(self):
3223 lostAndFoundBranches = set()
3225 user = gitConfig("git-p4.branchUser")
3226 if len(user) > 0:
3227 command = "branches -u %s" % user
3228 else:
3229 command = "branches"
3231 for info in p4CmdList(command):
3232 details = p4Cmd(["branch", "-o", info["branch"]])
3233 viewIdx = 0
3234 while "View%s" % viewIdx in details:
3235 paths = details["View%s" % viewIdx].split(" ")
3236 viewIdx = viewIdx + 1
3237 # require standard //depot/foo/... //depot/bar/... mapping
3238 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3239 continue
3240 source = paths[0]
3241 destination = paths[1]
3242 ## HACK
3243 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3244 source = source[len(self.depotPaths[0]):-4]
3245 destination = destination[len(self.depotPaths[0]):-4]
3247 if destination in self.knownBranches:
3248 if not self.silent:
3249 print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3250 print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3251 continue
3253 self.knownBranches[destination] = source
3255 lostAndFoundBranches.discard(destination)
3257 if source not in self.knownBranches:
3258 lostAndFoundBranches.add(source)
3260 # Perforce does not strictly require branches to be defined, so we also
3261 # check git config for a branch list.
3263 # Example of branch definition in git config file:
3264 # [git-p4]
3265 # branchList=main:branchA
3266 # branchList=main:branchB
3267 # branchList=branchA:branchC
3268 configBranches = gitConfigList("git-p4.branchList")
3269 for branch in configBranches:
3270 if branch:
3271 (source, destination) = branch.split(":")
3272 self.knownBranches[destination] = source
3274 lostAndFoundBranches.discard(destination)
3276 if source not in self.knownBranches:
3277 lostAndFoundBranches.add(source)
3280 for branch in lostAndFoundBranches:
3281 self.knownBranches[branch] = branch
3283 def getBranchMappingFromGitBranches(self):
3284 branches = p4BranchesInGit(self.importIntoRemotes)
3285 for branch in branches.keys():
3286 if branch == "master":
3287 branch = "main"
3288 else:
3289 branch = branch[len(self.projectName):]
3290 self.knownBranches[branch] = branch
3292 def updateOptionDict(self, d):
3293 option_keys = {}
3294 if self.keepRepoPath:
3295 option_keys['keepRepoPath'] = 1
3297 d["options"] = ' '.join(sorted(option_keys.keys()))
3299 def readOptions(self, d):
3300 self.keepRepoPath = ('options' in d
3301 and ('keepRepoPath' in d['options']))
3303 def gitRefForBranch(self, branch):
3304 if branch == "main":
3305 return self.refPrefix + "master"
3307 if len(branch) <= 0:
3308 return branch
3310 return self.refPrefix + self.projectName + branch
3312 def gitCommitByP4Change(self, ref, change):
3313 if self.verbose:
3314 print("looking in ref " + ref + " for change %s using bisect..." % change)
3316 earliestCommit = ""
3317 latestCommit = parseRevision(ref)
3319 while True:
3320 if self.verbose:
3321 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3322 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3323 if len(next) == 0:
3324 if self.verbose:
3325 print("argh")
3326 return ""
3327 log = extractLogMessageFromGitCommit(next)
3328 settings = extractSettingsGitLog(log)
3329 currentChange = int(settings['change'])
3330 if self.verbose:
3331 print("current change %s" % currentChange)
3333 if currentChange == change:
3334 if self.verbose:
3335 print("found %s" % next)
3336 return next
3338 if currentChange < change:
3339 earliestCommit = "^%s" % next
3340 else:
3341 if next == latestCommit:
3342 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3343 latestCommit = "%s^@" % next
3345 return ""
3347 def importNewBranch(self, branch, maxChange):
3348 # make fast-import flush all changes to disk and update the refs using the checkpoint
3349 # command so that we can try to find the branch parent in the git history
3350 self.gitStream.write("checkpoint\n\n");
3351 self.gitStream.flush();
3352 branchPrefix = self.depotPaths[0] + branch + "/"
3353 range = "@1,%s" % maxChange
3354 #print "prefix" + branchPrefix
3355 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3356 if len(changes) <= 0:
3357 return False
3358 firstChange = changes[0]
3359 #print "first change in branch: %s" % firstChange
3360 sourceBranch = self.knownBranches[branch]
3361 sourceDepotPath = self.depotPaths[0] + sourceBranch
3362 sourceRef = self.gitRefForBranch(sourceBranch)
3363 #print "source " + sourceBranch
3365 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3366 #print "branch parent: %s" % branchParentChange
3367 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3368 if len(gitParent) > 0:
3369 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3370 #print "parent git commit: %s" % gitParent
3372 self.importChanges(changes)
3373 return True
3375 def searchParent(self, parent, branch, target):
3376 parentFound = False
3377 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3378 "--no-merges", parent]):
3379 blob = blob.strip()
3380 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3381 parentFound = True
3382 if self.verbose:
3383 print("Found parent of %s in commit %s" % (branch, blob))
3384 break
3385 if parentFound:
3386 return blob
3387 else:
3388 return None
3390 def importChanges(self, changes, origin_revision=0):
3391 cnt = 1
3392 for change in changes:
3393 description = p4_describe(change)
3394 self.updateOptionDict(description)
3396 if not self.silent:
3397 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3398 sys.stdout.flush()
3399 cnt = cnt + 1
3401 try:
3402 if self.detectBranches:
3403 branches = self.splitFilesIntoBranches(description)
3404 for branch in branches.keys():
3405 ## HACK --hwn
3406 branchPrefix = self.depotPaths[0] + branch + "/"
3407 self.branchPrefixes = [ branchPrefix ]
3409 parent = ""
3411 filesForCommit = branches[branch]
3413 if self.verbose:
3414 print("branch is %s" % branch)
3416 self.updatedBranches.add(branch)
3418 if branch not in self.createdBranches:
3419 self.createdBranches.add(branch)
3420 parent = self.knownBranches[branch]
3421 if parent == branch:
3422 parent = ""
3423 else:
3424 fullBranch = self.projectName + branch
3425 if fullBranch not in self.p4BranchesInGit:
3426 if not self.silent:
3427 print("\n Importing new branch %s" % fullBranch);
3428 if self.importNewBranch(branch, change - 1):
3429 parent = ""
3430 self.p4BranchesInGit.append(fullBranch)
3431 if not self.silent:
3432 print("\n Resuming with change %s" % change);
3434 if self.verbose:
3435 print("parent determined through known branches: %s" % parent)
3437 branch = self.gitRefForBranch(branch)
3438 parent = self.gitRefForBranch(parent)
3440 if self.verbose:
3441 print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3443 if len(parent) == 0 and branch in self.initialParents:
3444 parent = self.initialParents[branch]
3445 del self.initialParents[branch]
3447 blob = None
3448 if len(parent) > 0:
3449 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3450 if self.verbose:
3451 print("Creating temporary branch: " + tempBranch)
3452 self.commit(description, filesForCommit, tempBranch)
3453 self.tempBranches.append(tempBranch)
3454 self.checkpoint()
3455 blob = self.searchParent(parent, branch, tempBranch)
3456 if blob:
3457 self.commit(description, filesForCommit, branch, blob)
3458 else:
3459 if self.verbose:
3460 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3461 self.commit(description, filesForCommit, branch, parent)
3462 else:
3463 files = self.extractFilesFromCommit(description)
3464 self.commit(description, files, self.branch,
3465 self.initialParent)
3466 # only needed once, to connect to the previous commit
3467 self.initialParent = ""
3468 except IOError:
3469 print(self.gitError.read())
3470 sys.exit(1)
3472 def sync_origin_only(self):
3473 if self.syncWithOrigin:
3474 self.hasOrigin = originP4BranchesExist()
3475 if self.hasOrigin:
3476 if not self.silent:
3477 print('Syncing with origin first, using "git fetch origin"')
3478 system("git fetch origin")
3480 def importHeadRevision(self, revision):
3481 print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3483 details = {}
3484 details["user"] = "git perforce import user"
3485 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3486 % (' '.join(self.depotPaths), revision))
3487 details["change"] = revision
3488 newestRevision = 0
3490 fileCnt = 0
3491 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3493 for info in p4CmdList(["files"] + fileArgs):
3495 if 'code' in info and info['code'] == 'error':
3496 sys.stderr.write("p4 returned an error: %s\n"
3497 % info['data'])
3498 if info['data'].find("must refer to client") >= 0:
3499 sys.stderr.write("This particular p4 error is misleading.\n")
3500 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3501 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3502 sys.exit(1)
3503 if 'p4ExitCode' in info:
3504 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3505 sys.exit(1)
3508 change = int(info["change"])
3509 if change > newestRevision:
3510 newestRevision = change
3512 if info["action"] in self.delete_actions:
3513 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3514 #fileCnt = fileCnt + 1
3515 continue
3517 for prop in ["depotFile", "rev", "action", "type" ]:
3518 details["%s%s" % (prop, fileCnt)] = info[prop]
3520 fileCnt = fileCnt + 1
3522 details["change"] = newestRevision
3524 # Use time from top-most change so that all git p4 clones of
3525 # the same p4 repo have the same commit SHA1s.
3526 res = p4_describe(newestRevision)
3527 details["time"] = res["time"]
3529 self.updateOptionDict(details)
3530 try:
3531 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3532 except IOError as err:
3533 print("IO error with git fast-import. Is your git version recent enough?")
3534 print("IO error details: {}".format(err))
3535 print(self.gitError.read())
3537 def openStreams(self):
3538 self.importProcess = subprocess.Popen(["git", "fast-import"],
3539 stdin=subprocess.PIPE,
3540 stdout=subprocess.PIPE,
3541 stderr=subprocess.PIPE);
3542 self.gitOutput = self.importProcess.stdout
3543 self.gitStream = self.importProcess.stdin
3544 self.gitError = self.importProcess.stderr
3546 def closeStreams(self):
3547 self.gitStream.close()
3548 if self.importProcess.wait() != 0:
3549 die("fast-import failed: %s" % self.gitError.read())
3550 self.gitOutput.close()
3551 self.gitError.close()
3553 def run(self, args):
3554 if self.importIntoRemotes:
3555 self.refPrefix = "refs/remotes/p4/"
3556 else:
3557 self.refPrefix = "refs/heads/p4/"
3559 self.sync_origin_only()
3561 branch_arg_given = bool(self.branch)
3562 if len(self.branch) == 0:
3563 self.branch = self.refPrefix + "master"
3564 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3565 system("git update-ref %s refs/heads/p4" % self.branch)
3566 system("git branch -D p4")
3568 # accept either the command-line option, or the configuration variable
3569 if self.useClientSpec:
3570 # will use this after clone to set the variable
3571 self.useClientSpec_from_options = True
3572 else:
3573 if gitConfigBool("git-p4.useclientspec"):
3574 self.useClientSpec = True
3575 if self.useClientSpec:
3576 self.clientSpecDirs = getClientSpec()
3578 # TODO: should always look at previous commits,
3579 # merge with previous imports, if possible.
3580 if args == []:
3581 if self.hasOrigin:
3582 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3584 # branches holds mapping from branch name to sha1
3585 branches = p4BranchesInGit(self.importIntoRemotes)
3587 # restrict to just this one, disabling detect-branches
3588 if branch_arg_given:
3589 short = self.branch.split("/")[-1]
3590 if short in branches:
3591 self.p4BranchesInGit = [ short ]
3592 else:
3593 self.p4BranchesInGit = branches.keys()
3595 if len(self.p4BranchesInGit) > 1:
3596 if not self.silent:
3597 print("Importing from/into multiple branches")
3598 self.detectBranches = True
3599 for branch in branches.keys():
3600 self.initialParents[self.refPrefix + branch] = \
3601 branches[branch]
3603 if self.verbose:
3604 print("branches: %s" % self.p4BranchesInGit)
3606 p4Change = 0
3607 for branch in self.p4BranchesInGit:
3608 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3610 settings = extractSettingsGitLog(logMsg)
3612 self.readOptions(settings)
3613 if ('depot-paths' in settings
3614 and 'change' in settings):
3615 change = int(settings['change']) + 1
3616 p4Change = max(p4Change, change)
3618 depotPaths = sorted(settings['depot-paths'])
3619 if self.previousDepotPaths == []:
3620 self.previousDepotPaths = depotPaths
3621 else:
3622 paths = []
3623 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3624 prev_list = prev.split("/")
3625 cur_list = cur.split("/")
3626 for i in range(0, min(len(cur_list), len(prev_list))):
3627 if cur_list[i] != prev_list[i]:
3628 i = i - 1
3629 break
3631 paths.append ("/".join(cur_list[:i + 1]))
3633 self.previousDepotPaths = paths
3635 if p4Change > 0:
3636 self.depotPaths = sorted(self.previousDepotPaths)
3637 self.changeRange = "@%s,#head" % p4Change
3638 if not self.silent and not self.detectBranches:
3639 print("Performing incremental import into %s git branch" % self.branch)
3641 # accept multiple ref name abbreviations:
3642 # refs/foo/bar/branch -> use it exactly
3643 # p4/branch -> prepend refs/remotes/ or refs/heads/
3644 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3645 if not self.branch.startswith("refs/"):
3646 if self.importIntoRemotes:
3647 prepend = "refs/remotes/"
3648 else:
3649 prepend = "refs/heads/"
3650 if not self.branch.startswith("p4/"):
3651 prepend += "p4/"
3652 self.branch = prepend + self.branch
3654 if len(args) == 0 and self.depotPaths:
3655 if not self.silent:
3656 print("Depot paths: %s" % ' '.join(self.depotPaths))
3657 else:
3658 if self.depotPaths and self.depotPaths != args:
3659 print("previous import used depot path %s and now %s was specified. "
3660 "This doesn't work!" % (' '.join (self.depotPaths),
3661 ' '.join (args)))
3662 sys.exit(1)
3664 self.depotPaths = sorted(args)
3666 revision = ""
3667 self.users = {}
3669 # Make sure no revision specifiers are used when --changesfile
3670 # is specified.
3671 bad_changesfile = False
3672 if len(self.changesFile) > 0:
3673 for p in self.depotPaths:
3674 if p.find("@") >= 0 or p.find("#") >= 0:
3675 bad_changesfile = True
3676 break
3677 if bad_changesfile:
3678 die("Option --changesfile is incompatible with revision specifiers")
3680 newPaths = []
3681 for p in self.depotPaths:
3682 if p.find("@") != -1:
3683 atIdx = p.index("@")
3684 self.changeRange = p[atIdx:]
3685 if self.changeRange == "@all":
3686 self.changeRange = ""
3687 elif ',' not in self.changeRange:
3688 revision = self.changeRange
3689 self.changeRange = ""
3690 p = p[:atIdx]
3691 elif p.find("#") != -1:
3692 hashIdx = p.index("#")
3693 revision = p[hashIdx:]
3694 p = p[:hashIdx]
3695 elif self.previousDepotPaths == []:
3696 # pay attention to changesfile, if given, else import
3697 # the entire p4 tree at the head revision
3698 if len(self.changesFile) == 0:
3699 revision = "#head"
3701 p = re.sub ("\.\.\.$", "", p)
3702 if not p.endswith("/"):
3703 p += "/"
3705 newPaths.append(p)
3707 self.depotPaths = newPaths
3709 # --detect-branches may change this for each branch
3710 self.branchPrefixes = self.depotPaths
3712 self.loadUserMapFromCache()
3713 self.labels = {}
3714 if self.detectLabels:
3715 self.getLabels();
3717 if self.detectBranches:
3718 ## FIXME - what's a P4 projectName ?
3719 self.projectName = self.guessProjectName()
3721 if self.hasOrigin:
3722 self.getBranchMappingFromGitBranches()
3723 else:
3724 self.getBranchMapping()
3725 if self.verbose:
3726 print("p4-git branches: %s" % self.p4BranchesInGit)
3727 print("initial parents: %s" % self.initialParents)
3728 for b in self.p4BranchesInGit:
3729 if b != "master":
3731 ## FIXME
3732 b = b[len(self.projectName):]
3733 self.createdBranches.add(b)
3735 self.openStreams()
3737 if revision:
3738 self.importHeadRevision(revision)
3739 else:
3740 changes = []
3742 if len(self.changesFile) > 0:
3743 output = open(self.changesFile).readlines()
3744 changeSet = set()
3745 for line in output:
3746 changeSet.add(int(line))
3748 for change in changeSet:
3749 changes.append(change)
3751 changes.sort()
3752 else:
3753 # catch "git p4 sync" with no new branches, in a repo that
3754 # does not have any existing p4 branches
3755 if len(args) == 0:
3756 if not self.p4BranchesInGit:
3757 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3759 # The default branch is master, unless --branch is used to
3760 # specify something else. Make sure it exists, or complain
3761 # nicely about how to use --branch.
3762 if not self.detectBranches:
3763 if not branch_exists(self.branch):
3764 if branch_arg_given:
3765 die("Error: branch %s does not exist." % self.branch)
3766 else:
3767 die("Error: no branch %s; perhaps specify one with --branch." %
3768 self.branch)
3770 if self.verbose:
3771 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3772 self.changeRange))
3773 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3775 if len(self.maxChanges) > 0:
3776 changes = changes[:min(int(self.maxChanges), len(changes))]
3778 if len(changes) == 0:
3779 if not self.silent:
3780 print("No changes to import!")
3781 else:
3782 if not self.silent and not self.detectBranches:
3783 print("Import destination: %s" % self.branch)
3785 self.updatedBranches = set()
3787 if not self.detectBranches:
3788 if args:
3789 # start a new branch
3790 self.initialParent = ""
3791 else:
3792 # build on a previous revision
3793 self.initialParent = parseRevision(self.branch)
3795 self.importChanges(changes)
3797 if not self.silent:
3798 print("")
3799 if len(self.updatedBranches) > 0:
3800 sys.stdout.write("Updated branches: ")
3801 for b in self.updatedBranches:
3802 sys.stdout.write("%s " % b)
3803 sys.stdout.write("\n")
3805 if gitConfigBool("git-p4.importLabels"):
3806 self.importLabels = True
3808 if self.importLabels:
3809 p4Labels = getP4Labels(self.depotPaths)
3810 gitTags = getGitTags()
3812 missingP4Labels = p4Labels - gitTags
3813 self.importP4Labels(self.gitStream, missingP4Labels)
3815 self.closeStreams()
3817 # Cleanup temporary branches created during import
3818 if self.tempBranches != []:
3819 for branch in self.tempBranches:
3820 read_pipe("git update-ref -d %s" % branch)
3821 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3823 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3824 # a convenient shortcut refname "p4".
3825 if self.importIntoRemotes:
3826 head_ref = self.refPrefix + "HEAD"
3827 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3828 system(["git", "symbolic-ref", head_ref, self.branch])
3830 return True
3832 class P4Rebase(Command):
3833 def __init__(self):
3834 Command.__init__(self)
3835 self.options = [
3836 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3838 self.importLabels = False
3839 self.description = ("Fetches the latest revision from perforce and "
3840 + "rebases the current work (branch) against it")
3842 def run(self, args):
3843 sync = P4Sync()
3844 sync.importLabels = self.importLabels
3845 sync.run([])
3847 return self.rebase()
3849 def rebase(self):
3850 if os.system("git update-index --refresh") != 0:
3851 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3852 if len(read_pipe("git diff-index HEAD --")) > 0:
3853 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3855 [upstream, settings] = findUpstreamBranchPoint()
3856 if len(upstream) == 0:
3857 die("Cannot find upstream branchpoint for rebase")
3859 # the branchpoint may be p4/foo~3, so strip off the parent
3860 upstream = re.sub("~[0-9]+$", "", upstream)
3862 print("Rebasing the current branch onto %s" % upstream)
3863 oldHead = read_pipe("git rev-parse HEAD").strip()
3864 system("git rebase %s" % upstream)
3865 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3866 return True
3868 class P4Clone(P4Sync):
3869 def __init__(self):
3870 P4Sync.__init__(self)
3871 self.description = "Creates a new git repository and imports from Perforce into it"
3872 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3873 self.options += [
3874 optparse.make_option("--destination", dest="cloneDestination",
3875 action='store', default=None,
3876 help="where to leave result of the clone"),
3877 optparse.make_option("--bare", dest="cloneBare",
3878 action="store_true", default=False),
3880 self.cloneDestination = None
3881 self.needsGit = False
3882 self.cloneBare = False
3884 def defaultDestination(self, args):
3885 ## TODO: use common prefix of args?
3886 depotPath = args[0]
3887 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3888 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3889 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3890 depotDir = re.sub(r"/$", "", depotDir)
3891 return os.path.split(depotDir)[1]
3893 def run(self, args):
3894 if len(args) < 1:
3895 return False
3897 if self.keepRepoPath and not self.cloneDestination:
3898 sys.stderr.write("Must specify destination for --keep-path\n")
3899 sys.exit(1)
3901 depotPaths = args
3903 if not self.cloneDestination and len(depotPaths) > 1:
3904 self.cloneDestination = depotPaths[-1]
3905 depotPaths = depotPaths[:-1]
3907 for p in depotPaths:
3908 if not p.startswith("//"):
3909 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3910 return False
3912 if not self.cloneDestination:
3913 self.cloneDestination = self.defaultDestination(args)
3915 print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3917 if not os.path.exists(self.cloneDestination):
3918 os.makedirs(self.cloneDestination)
3919 chdir(self.cloneDestination)
3921 init_cmd = [ "git", "init" ]
3922 if self.cloneBare:
3923 init_cmd.append("--bare")
3924 retcode = subprocess.call(init_cmd)
3925 if retcode:
3926 raise CalledProcessError(retcode, init_cmd)
3928 if not P4Sync.run(self, depotPaths):
3929 return False
3931 # create a master branch and check out a work tree
3932 if gitBranchExists(self.branch):
3933 system([ "git", "branch", "master", self.branch ])
3934 if not self.cloneBare:
3935 system([ "git", "checkout", "-f" ])
3936 else:
3937 print('Not checking out any branch, use ' \
3938 '"git checkout -q -b master <branch>"')
3940 # auto-set this variable if invoked with --use-client-spec
3941 if self.useClientSpec_from_options:
3942 system("git config --bool git-p4.useclientspec true")
3944 return True
3946 class P4Unshelve(Command):
3947 def __init__(self):
3948 Command.__init__(self)
3949 self.options = []
3950 self.origin = "HEAD"
3951 self.description = "Unshelve a P4 changelist into a git commit"
3952 self.usage = "usage: %prog [options] changelist"
3953 self.options += [
3954 optparse.make_option("--origin", dest="origin",
3955 help="Use this base revision instead of the default (%s)" % self.origin),
3957 self.verbose = False
3958 self.noCommit = False
3959 self.destbranch = "refs/remotes/p4-unshelved"
3961 def renameBranch(self, branch_name):
3962 """ Rename the existing branch to branch_name.N
3965 found = True
3966 for i in range(0,1000):
3967 backup_branch_name = "{0}.{1}".format(branch_name, i)
3968 if not gitBranchExists(backup_branch_name):
3969 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3970 gitDeleteRef(branch_name)
3971 found = True
3972 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3973 break
3975 if not found:
3976 sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3978 def findLastP4Revision(self, starting_point):
3979 """ Look back from starting_point for the first commit created by git-p4
3980 to find the P4 commit we are based on, and the depot-paths.
3983 for parent in (range(65535)):
3984 log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3985 settings = extractSettingsGitLog(log)
3986 if 'change' in settings:
3987 return settings
3989 sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3991 def createShelveParent(self, change, branch_name, sync, origin):
3992 """ Create a commit matching the parent of the shelved changelist 'change'
3994 parent_description = p4_describe(change, shelved=True)
3995 parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
3996 files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
3998 parent_files = []
3999 for f in files:
4000 # if it was added in the shelved changelist, it won't exist in the parent
4001 if f['action'] in self.add_actions:
4002 continue
4004 # if it was deleted in the shelved changelist it must not be deleted
4005 # in the parent - we might even need to create it if the origin branch
4006 # does not have it
4007 if f['action'] in self.delete_actions:
4008 f['action'] = 'add'
4010 parent_files.append(f)
4012 sync.commit(parent_description, parent_files, branch_name,
4013 parent=origin, allow_empty=True)
4014 print("created parent commit for {0} based on {1} in {2}".format(
4015 change, self.origin, branch_name))
4017 def run(self, args):
4018 if len(args) != 1:
4019 return False
4021 if not gitBranchExists(self.origin):
4022 sys.exit("origin branch {0} does not exist".format(self.origin))
4024 sync = P4Sync()
4025 changes = args
4027 # only one change at a time
4028 change = changes[0]
4030 # if the target branch already exists, rename it
4031 branch_name = "{0}/{1}".format(self.destbranch, change)
4032 if gitBranchExists(branch_name):
4033 self.renameBranch(branch_name)
4034 sync.branch = branch_name
4036 sync.verbose = self.verbose
4037 sync.suppress_meta_comment = True
4039 settings = self.findLastP4Revision(self.origin)
4040 sync.depotPaths = settings['depot-paths']
4041 sync.branchPrefixes = sync.depotPaths
4043 sync.openStreams()
4044 sync.loadUserMapFromCache()
4045 sync.silent = True
4047 # create a commit for the parent of the shelved changelist
4048 self.createShelveParent(change, branch_name, sync, self.origin)
4050 # create the commit for the shelved changelist itself
4051 description = p4_describe(change, True)
4052 files = sync.extractFilesFromCommit(description, True, change)
4054 sync.commit(description, files, branch_name, "")
4055 sync.closeStreams()
4057 print("unshelved changelist {0} into {1}".format(change, branch_name))
4059 return True
4061 class P4Branches(Command):
4062 def __init__(self):
4063 Command.__init__(self)
4064 self.options = [ ]
4065 self.description = ("Shows the git branches that hold imports and their "
4066 + "corresponding perforce depot paths")
4067 self.verbose = False
4069 def run(self, args):
4070 if originP4BranchesExist():
4071 createOrUpdateBranchesFromOrigin()
4073 cmdline = "git rev-parse --symbolic "
4074 cmdline += " --remotes"
4076 for line in read_pipe_lines(cmdline):
4077 line = line.strip()
4079 if not line.startswith('p4/') or line == "p4/HEAD":
4080 continue
4081 branch = line
4083 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4084 settings = extractSettingsGitLog(log)
4086 print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4087 return True
4089 class HelpFormatter(optparse.IndentedHelpFormatter):
4090 def __init__(self):
4091 optparse.IndentedHelpFormatter.__init__(self)
4093 def format_description(self, description):
4094 if description:
4095 return description + "\n"
4096 else:
4097 return ""
4099 def printUsage(commands):
4100 print("usage: %s <command> [options]" % sys.argv[0])
4101 print("")
4102 print("valid commands: %s" % ", ".join(commands))
4103 print("")
4104 print("Try %s <command> --help for command specific help." % sys.argv[0])
4105 print("")
4107 commands = {
4108 "debug" : P4Debug,
4109 "submit" : P4Submit,
4110 "commit" : P4Submit,
4111 "sync" : P4Sync,
4112 "rebase" : P4Rebase,
4113 "clone" : P4Clone,
4114 "rollback" : P4RollBack,
4115 "branches" : P4Branches,
4116 "unshelve" : P4Unshelve,
4120 def main():
4121 if len(sys.argv[1:]) == 0:
4122 printUsage(commands.keys())
4123 sys.exit(2)
4125 cmdName = sys.argv[1]
4126 try:
4127 klass = commands[cmdName]
4128 cmd = klass()
4129 except KeyError:
4130 print("unknown command %s" % cmdName)
4131 print("")
4132 printUsage(commands.keys())
4133 sys.exit(2)
4135 options = cmd.options
4136 cmd.gitdir = os.environ.get("GIT_DIR", None)
4138 args = sys.argv[2:]
4140 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4141 if cmd.needsGit:
4142 options.append(optparse.make_option("--git-dir", dest="gitdir"))
4144 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4145 options,
4146 description = cmd.description,
4147 formatter = HelpFormatter())
4149 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4150 global verbose
4151 verbose = cmd.verbose
4152 if cmd.needsGit:
4153 if cmd.gitdir == None:
4154 cmd.gitdir = os.path.abspath(".git")
4155 if not isValidGitDir(cmd.gitdir):
4156 # "rev-parse --git-dir" without arguments will try $PWD/.git
4157 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4158 if os.path.exists(cmd.gitdir):
4159 cdup = read_pipe("git rev-parse --show-cdup").strip()
4160 if len(cdup) > 0:
4161 chdir(cdup);
4163 if not isValidGitDir(cmd.gitdir):
4164 if isValidGitDir(cmd.gitdir + "/.git"):
4165 cmd.gitdir += "/.git"
4166 else:
4167 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4169 # so git commands invoked from the P4 workspace will succeed
4170 os.environ["GIT_DIR"] = cmd.gitdir
4172 if not cmd.run(args):
4173 parser.print_help()
4174 sys.exit(2)
4177 if __name__ == '__main__':
4178 main()