Merge branch 'ds/commit-graph-assert-missing-parents'
[git/gitster.git] / git-p4.py
blob1998c3e141bec5142032f08623b916d7741e64ac
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
30 # support basestring in python3
31 try:
32 unicode = unicode
33 except NameError:
34 # 'unicode' is undefined, must be Python 3
35 str = str
36 unicode = str
37 bytes = bytes
38 basestring = (str,bytes)
39 else:
40 # 'unicode' exists, must be Python 2
41 str = str
42 unicode = unicode
43 bytes = str
44 basestring = basestring
46 try:
47 from subprocess import CalledProcessError
48 except ImportError:
49 # from python2.7:subprocess.py
50 # Exception classes used by this module.
51 class CalledProcessError(Exception):
52 """This exception is raised when a process run by check_call() returns
53 a non-zero exit status. The exit status will be stored in the
54 returncode attribute."""
55 def __init__(self, returncode, cmd):
56 self.returncode = returncode
57 self.cmd = cmd
58 def __str__(self):
59 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
61 verbose = False
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
69 p4_access_checked = False
71 def p4_build_cmd(cmd):
72 """Build a suitable p4 command line.
74 This consolidates building and returning a p4 command line into one
75 location. It means that hooking into the environment, or other configuration
76 can be done more easily.
77 """
78 real_cmd = ["p4"]
80 user = gitConfig("git-p4.user")
81 if len(user) > 0:
82 real_cmd += ["-u",user]
84 password = gitConfig("git-p4.password")
85 if len(password) > 0:
86 real_cmd += ["-P", password]
88 port = gitConfig("git-p4.port")
89 if len(port) > 0:
90 real_cmd += ["-p", port]
92 host = gitConfig("git-p4.host")
93 if len(host) > 0:
94 real_cmd += ["-H", host]
96 client = gitConfig("git-p4.client")
97 if len(client) > 0:
98 real_cmd += ["-c", client]
100 retries = gitConfigInt("git-p4.retries")
101 if retries is None:
102 # Perform 3 retries by default
103 retries = 3
104 if retries > 0:
105 # Provide a way to not pass this option by setting git-p4.retries to 0
106 real_cmd += ["-r", str(retries)]
108 if isinstance(cmd,basestring):
109 real_cmd = ' '.join(real_cmd) + ' ' + cmd
110 else:
111 real_cmd += cmd
113 # now check that we can actually talk to the server
114 global p4_access_checked
115 if not p4_access_checked:
116 p4_access_checked = True # suppress access checks in p4_check_access itself
117 p4_check_access()
119 return real_cmd
121 def git_dir(path):
122 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123 This won't automatically add ".git" to a directory.
125 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126 if not d or len(d) == 0:
127 return None
128 else:
129 return d
131 def chdir(path, is_client_path=False):
132 """Do chdir to the given path, and set the PWD environment
133 variable for use by P4. It does not look at getcwd() output.
134 Since we're not using the shell, it is necessary to set the
135 PWD environment variable explicitly.
137 Normally, expand the path to force it to be absolute. This
138 addresses the use of relative path names inside P4 settings,
139 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
140 as given; it looks for .p4config using PWD.
142 If is_client_path, the path was handed to us directly by p4,
143 and may be a symbolic link. Do not call os.getcwd() in this
144 case, because it will cause p4 to think that PWD is not inside
145 the client path.
148 os.chdir(path)
149 if not is_client_path:
150 path = os.getcwd()
151 os.environ['PWD'] = path
153 def calcDiskFree():
154 """Return free space in bytes on the disk of the given dirname."""
155 if platform.system() == 'Windows':
156 free_bytes = ctypes.c_ulonglong(0)
157 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158 return free_bytes.value
159 else:
160 st = os.statvfs(os.getcwd())
161 return st.f_bavail * st.f_frsize
163 def die(msg):
164 if verbose:
165 raise Exception(msg)
166 else:
167 sys.stderr.write(msg + "\n")
168 sys.exit(1)
170 def write_pipe(c, stdin):
171 if verbose:
172 sys.stderr.write('Writing pipe: %s\n' % str(c))
174 expand = isinstance(c,basestring)
175 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176 pipe = p.stdin
177 val = pipe.write(stdin)
178 pipe.close()
179 if p.wait():
180 die('Command failed: %s' % str(c))
182 return val
184 def p4_write_pipe(c, stdin):
185 real_cmd = p4_build_cmd(c)
186 return write_pipe(real_cmd, stdin)
188 def read_pipe_full(c):
189 """ Read output from command. Returns a tuple
190 of the return status, stdout text and stderr
191 text.
193 if verbose:
194 sys.stderr.write('Reading pipe: %s\n' % str(c))
196 expand = isinstance(c,basestring)
197 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198 (out, err) = p.communicate()
199 return (p.returncode, out, err)
201 def read_pipe(c, ignore_error=False):
202 """ Read output from command. Returns the output text on
203 success. On failure, terminates execution, unless
204 ignore_error is True, when it returns an empty string.
206 (retcode, out, err) = read_pipe_full(c)
207 if retcode != 0:
208 if ignore_error:
209 out = ""
210 else:
211 die('Command failed: %s\nError: %s' % (str(c), err))
212 return out
214 def read_pipe_text(c):
215 """ Read output from a command with trailing whitespace stripped.
216 On error, returns None.
218 (retcode, out, err) = read_pipe_full(c)
219 if retcode != 0:
220 return None
221 else:
222 return out.rstrip()
224 def p4_read_pipe(c, ignore_error=False):
225 real_cmd = p4_build_cmd(c)
226 return read_pipe(real_cmd, ignore_error)
228 def read_pipe_lines(c):
229 if verbose:
230 sys.stderr.write('Reading pipe: %s\n' % str(c))
232 expand = isinstance(c, basestring)
233 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234 pipe = p.stdout
235 val = pipe.readlines()
236 if pipe.close() or p.wait():
237 die('Command failed: %s' % str(c))
239 return val
241 def p4_read_pipe_lines(c):
242 """Specifically invoke p4 on the command supplied. """
243 real_cmd = p4_build_cmd(c)
244 return read_pipe_lines(real_cmd)
246 def p4_has_command(cmd):
247 """Ask p4 for help on this command. If it returns an error, the
248 command does not exist in this version of p4."""
249 real_cmd = p4_build_cmd(["help", cmd])
250 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251 stderr=subprocess.PIPE)
252 p.communicate()
253 return p.returncode == 0
255 def p4_has_move_command():
256 """See if the move command exists, that it supports -k, and that
257 it has not been administratively disabled. The arguments
258 must be correct, but the filenames do not have to exist. Use
259 ones with wildcards so even if they exist, it will fail."""
261 if not p4_has_command("move"):
262 return False
263 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265 (out, err) = p.communicate()
266 # return code will be 1 in either case
267 if err.find("Invalid option") >= 0:
268 return False
269 if err.find("disabled") >= 0:
270 return False
271 # assume it failed because @... was invalid changelist
272 return True
274 def system(cmd, ignore_error=False):
275 expand = isinstance(cmd,basestring)
276 if verbose:
277 sys.stderr.write("executing %s\n" % str(cmd))
278 retcode = subprocess.call(cmd, shell=expand)
279 if retcode and not ignore_error:
280 raise CalledProcessError(retcode, cmd)
282 return retcode
284 def p4_system(cmd):
285 """Specifically invoke p4 as the system command. """
286 real_cmd = p4_build_cmd(cmd)
287 expand = isinstance(real_cmd, basestring)
288 retcode = subprocess.call(real_cmd, shell=expand)
289 if retcode:
290 raise CalledProcessError(retcode, real_cmd)
292 def die_bad_access(s):
293 die("failure accessing depot: {0}".format(s.rstrip()))
295 def p4_check_access(min_expiration=1):
296 """ Check if we can access Perforce - account still logged in
298 results = p4CmdList(["login", "-s"])
300 if len(results) == 0:
301 # should never get here: always get either some results, or a p4ExitCode
302 assert("could not parse response from perforce")
304 result = results[0]
306 if 'p4ExitCode' in result:
307 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308 die_bad_access("could not run p4")
310 code = result.get("code")
311 if not code:
312 # we get here if we couldn't connect and there was nothing to unmarshal
313 die_bad_access("could not connect")
315 elif code == "stat":
316 expiry = result.get("TicketExpiration")
317 if expiry:
318 expiry = int(expiry)
319 if expiry > min_expiration:
320 # ok to carry on
321 return
322 else:
323 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
325 else:
326 # account without a timeout - all ok
327 return
329 elif code == "error":
330 data = result.get("data")
331 if data:
332 die_bad_access("p4 error: {0}".format(data))
333 else:
334 die_bad_access("unknown error")
335 else:
336 die_bad_access("unknown error code {0}".format(code))
338 _p4_version_string = None
339 def p4_version_string():
340 """Read the version string, showing just the last line, which
341 hopefully is the interesting version bit.
343 $ p4 -V
344 Perforce - The Fast Software Configuration Management System.
345 Copyright 1995-2011 Perforce Software. All rights reserved.
346 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
348 global _p4_version_string
349 if not _p4_version_string:
350 a = p4_read_pipe_lines(["-V"])
351 _p4_version_string = a[-1].rstrip()
352 return _p4_version_string
354 def p4_integrate(src, dest):
355 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
357 def p4_sync(f, *options):
358 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
360 def p4_add(f):
361 # forcibly add file names with wildcards
362 if wildcard_present(f):
363 p4_system(["add", "-f", f])
364 else:
365 p4_system(["add", f])
367 def p4_delete(f):
368 p4_system(["delete", wildcard_encode(f)])
370 def p4_edit(f, *options):
371 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
373 def p4_revert(f):
374 p4_system(["revert", wildcard_encode(f)])
376 def p4_reopen(type, f):
377 p4_system(["reopen", "-t", type, wildcard_encode(f)])
379 def p4_reopen_in_change(changelist, files):
380 cmd = ["reopen", "-c", str(changelist)] + files
381 p4_system(cmd)
383 def p4_move(src, dest):
384 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
386 def p4_last_change():
387 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
388 return int(results[0]['change'])
390 def p4_describe(change, shelved=False):
391 """Make sure it returns a valid result by checking for
392 the presence of field "time". Return a dict of the
393 results."""
395 cmd = ["describe", "-s"]
396 if shelved:
397 cmd += ["-S"]
398 cmd += [str(change)]
400 ds = p4CmdList(cmd, skip_info=True)
401 if len(ds) != 1:
402 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
404 d = ds[0]
406 if "p4ExitCode" in d:
407 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
408 str(d)))
409 if "code" in d:
410 if d["code"] == "error":
411 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
413 if "time" not in d:
414 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
416 return d
419 # Canonicalize the p4 type and return a tuple of the
420 # base type, plus any modifiers. See "p4 help filetypes"
421 # for a list and explanation.
423 def split_p4_type(p4type):
425 p4_filetypes_historical = {
426 "ctempobj": "binary+Sw",
427 "ctext": "text+C",
428 "cxtext": "text+Cx",
429 "ktext": "text+k",
430 "kxtext": "text+kx",
431 "ltext": "text+F",
432 "tempobj": "binary+FSw",
433 "ubinary": "binary+F",
434 "uresource": "resource+F",
435 "uxbinary": "binary+Fx",
436 "xbinary": "binary+x",
437 "xltext": "text+Fx",
438 "xtempobj": "binary+Swx",
439 "xtext": "text+x",
440 "xunicode": "unicode+x",
441 "xutf16": "utf16+x",
443 if p4type in p4_filetypes_historical:
444 p4type = p4_filetypes_historical[p4type]
445 mods = ""
446 s = p4type.split("+")
447 base = s[0]
448 mods = ""
449 if len(s) > 1:
450 mods = s[1]
451 return (base, mods)
454 # return the raw p4 type of a file (text, text+ko, etc)
456 def p4_type(f):
457 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
458 return results[0]['headType']
461 # Given a type base and modifier, return a regexp matching
462 # the keywords that can be expanded in the file
464 def p4_keywords_regexp_for_type(base, type_mods):
465 if base in ("text", "unicode", "binary"):
466 kwords = None
467 if "ko" in type_mods:
468 kwords = 'Id|Header'
469 elif "k" in type_mods:
470 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
471 else:
472 return None
473 pattern = r"""
474 \$ # Starts with a dollar, followed by...
475 (%s) # one of the keywords, followed by...
476 (:[^$\n]+)? # possibly an old expansion, followed by...
477 \$ # another dollar
478 """ % kwords
479 return pattern
480 else:
481 return None
484 # Given a file, return a regexp matching the possible
485 # RCS keywords that will be expanded, or None for files
486 # with kw expansion turned off.
488 def p4_keywords_regexp_for_file(file):
489 if not os.path.exists(file):
490 return None
491 else:
492 (type_base, type_mods) = split_p4_type(p4_type(file))
493 return p4_keywords_regexp_for_type(type_base, type_mods)
495 def setP4ExecBit(file, mode):
496 # Reopens an already open file and changes the execute bit to match
497 # the execute bit setting in the passed in mode.
499 p4Type = "+x"
501 if not isModeExec(mode):
502 p4Type = getP4OpenedType(file)
503 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
504 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
505 if p4Type[-1] == "+":
506 p4Type = p4Type[0:-1]
508 p4_reopen(p4Type, file)
510 def getP4OpenedType(file):
511 # Returns the perforce file type for the given file.
513 result = p4_read_pipe(["opened", wildcard_encode(file)])
514 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
515 if match:
516 return match.group(1)
517 else:
518 die("Could not determine file type for %s (result: '%s')" % (file, result))
520 # Return the set of all p4 labels
521 def getP4Labels(depotPaths):
522 labels = set()
523 if isinstance(depotPaths,basestring):
524 depotPaths = [depotPaths]
526 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
527 label = l['label']
528 labels.add(label)
530 return labels
532 # Return the set of all git tags
533 def getGitTags():
534 gitTags = set()
535 for line in read_pipe_lines(["git", "tag"]):
536 tag = line.strip()
537 gitTags.add(tag)
538 return gitTags
540 def diffTreePattern():
541 # This is a simple generator for the diff tree regex pattern. This could be
542 # a class variable if this and parseDiffTreeEntry were a part of a class.
543 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
544 while True:
545 yield pattern
547 def parseDiffTreeEntry(entry):
548 """Parses a single diff tree entry into its component elements.
550 See git-diff-tree(1) manpage for details about the format of the diff
551 output. This method returns a dictionary with the following elements:
553 src_mode - The mode of the source file
554 dst_mode - The mode of the destination file
555 src_sha1 - The sha1 for the source file
556 dst_sha1 - The sha1 fr the destination file
557 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
558 status_score - The score for the status (applicable for 'C' and 'R'
559 statuses). This is None if there is no score.
560 src - The path for the source file.
561 dst - The path for the destination file. This is only present for
562 copy or renames. If it is not present, this is None.
564 If the pattern is not matched, None is returned."""
566 match = diffTreePattern().next().match(entry)
567 if match:
568 return {
569 'src_mode': match.group(1),
570 'dst_mode': match.group(2),
571 'src_sha1': match.group(3),
572 'dst_sha1': match.group(4),
573 'status': match.group(5),
574 'status_score': match.group(6),
575 'src': match.group(7),
576 'dst': match.group(10)
578 return None
580 def isModeExec(mode):
581 # Returns True if the given git mode represents an executable file,
582 # otherwise False.
583 return mode[-3:] == "755"
585 class P4Exception(Exception):
586 """ Base class for exceptions from the p4 client """
587 def __init__(self, exit_code):
588 self.p4ExitCode = exit_code
590 class P4ServerException(P4Exception):
591 """ Base class for exceptions where we get some kind of marshalled up result from the server """
592 def __init__(self, exit_code, p4_result):
593 super(P4ServerException, self).__init__(exit_code)
594 self.p4_result = p4_result
595 self.code = p4_result[0]['code']
596 self.data = p4_result[0]['data']
598 class P4RequestSizeException(P4ServerException):
599 """ One of the maxresults or maxscanrows errors """
600 def __init__(self, exit_code, p4_result, limit):
601 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
602 self.limit = limit
604 def isModeExecChanged(src_mode, dst_mode):
605 return isModeExec(src_mode) != isModeExec(dst_mode)
607 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
608 errors_as_exceptions=False):
610 if isinstance(cmd,basestring):
611 cmd = "-G " + cmd
612 expand = True
613 else:
614 cmd = ["-G"] + cmd
615 expand = False
617 cmd = p4_build_cmd(cmd)
618 if verbose:
619 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
621 # Use a temporary file to avoid deadlocks without
622 # subprocess.communicate(), which would put another copy
623 # of stdout into memory.
624 stdin_file = None
625 if stdin is not None:
626 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
627 if isinstance(stdin,basestring):
628 stdin_file.write(stdin)
629 else:
630 for i in stdin:
631 stdin_file.write(i + '\n')
632 stdin_file.flush()
633 stdin_file.seek(0)
635 p4 = subprocess.Popen(cmd,
636 shell=expand,
637 stdin=stdin_file,
638 stdout=subprocess.PIPE)
640 result = []
641 try:
642 while True:
643 entry = marshal.load(p4.stdout)
644 if skip_info:
645 if 'code' in entry and entry['code'] == 'info':
646 continue
647 if cb is not None:
648 cb(entry)
649 else:
650 result.append(entry)
651 except EOFError:
652 pass
653 exitCode = p4.wait()
654 if exitCode != 0:
655 if errors_as_exceptions:
656 if len(result) > 0:
657 data = result[0].get('data')
658 if data:
659 m = re.search('Too many rows scanned \(over (\d+)\)', data)
660 if not m:
661 m = re.search('Request too large \(over (\d+)\)', data)
663 if m:
664 limit = int(m.group(1))
665 raise P4RequestSizeException(exitCode, result, limit)
667 raise P4ServerException(exitCode, result)
668 else:
669 raise P4Exception(exitCode)
670 else:
671 entry = {}
672 entry["p4ExitCode"] = exitCode
673 result.append(entry)
675 return result
677 def p4Cmd(cmd):
678 list = p4CmdList(cmd)
679 result = {}
680 for entry in list:
681 result.update(entry)
682 return result;
684 def p4Where(depotPath):
685 if not depotPath.endswith("/"):
686 depotPath += "/"
687 depotPathLong = depotPath + "..."
688 outputList = p4CmdList(["where", depotPathLong])
689 output = None
690 for entry in outputList:
691 if "depotFile" in entry:
692 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
693 # The base path always ends with "/...".
694 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
695 output = entry
696 break
697 elif "data" in entry:
698 data = entry.get("data")
699 space = data.find(" ")
700 if data[:space] == depotPath:
701 output = entry
702 break
703 if output == None:
704 return ""
705 if output["code"] == "error":
706 return ""
707 clientPath = ""
708 if "path" in output:
709 clientPath = output.get("path")
710 elif "data" in output:
711 data = output.get("data")
712 lastSpace = data.rfind(" ")
713 clientPath = data[lastSpace + 1:]
715 if clientPath.endswith("..."):
716 clientPath = clientPath[:-3]
717 return clientPath
719 def currentGitBranch():
720 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
722 def isValidGitDir(path):
723 return git_dir(path) != None
725 def parseRevision(ref):
726 return read_pipe("git rev-parse %s" % ref).strip()
728 def branchExists(ref):
729 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
730 ignore_error=True)
731 return len(rev) > 0
733 def extractLogMessageFromGitCommit(commit):
734 logMessage = ""
736 ## fixme: title is first line of commit, not 1st paragraph.
737 foundTitle = False
738 for log in read_pipe_lines("git cat-file commit %s" % commit):
739 if not foundTitle:
740 if len(log) == 1:
741 foundTitle = True
742 continue
744 logMessage += log
745 return logMessage
747 def extractSettingsGitLog(log):
748 values = {}
749 for line in log.split("\n"):
750 line = line.strip()
751 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
752 if not m:
753 continue
755 assignments = m.group(1).split (':')
756 for a in assignments:
757 vals = a.split ('=')
758 key = vals[0].strip()
759 val = ('='.join (vals[1:])).strip()
760 if val.endswith ('\"') and val.startswith('"'):
761 val = val[1:-1]
763 values[key] = val
765 paths = values.get("depot-paths")
766 if not paths:
767 paths = values.get("depot-path")
768 if paths:
769 values['depot-paths'] = paths.split(',')
770 return values
772 def gitBranchExists(branch):
773 proc = subprocess.Popen(["git", "rev-parse", branch],
774 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
775 return proc.wait() == 0;
777 def gitUpdateRef(ref, newvalue):
778 subprocess.check_call(["git", "update-ref", ref, newvalue])
780 def gitDeleteRef(ref):
781 subprocess.check_call(["git", "update-ref", "-d", ref])
783 _gitConfig = {}
785 def gitConfig(key, typeSpecifier=None):
786 if key not in _gitConfig:
787 cmd = [ "git", "config" ]
788 if typeSpecifier:
789 cmd += [ typeSpecifier ]
790 cmd += [ key ]
791 s = read_pipe(cmd, ignore_error=True)
792 _gitConfig[key] = s.strip()
793 return _gitConfig[key]
795 def gitConfigBool(key):
796 """Return a bool, using git config --bool. It is True only if the
797 variable is set to true, and False if set to false or not present
798 in the config."""
800 if key not in _gitConfig:
801 _gitConfig[key] = gitConfig(key, '--bool') == "true"
802 return _gitConfig[key]
804 def gitConfigInt(key):
805 if key not in _gitConfig:
806 cmd = [ "git", "config", "--int", key ]
807 s = read_pipe(cmd, ignore_error=True)
808 v = s.strip()
809 try:
810 _gitConfig[key] = int(gitConfig(key, '--int'))
811 except ValueError:
812 _gitConfig[key] = None
813 return _gitConfig[key]
815 def gitConfigList(key):
816 if key not in _gitConfig:
817 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
818 _gitConfig[key] = s.strip().splitlines()
819 if _gitConfig[key] == ['']:
820 _gitConfig[key] = []
821 return _gitConfig[key]
823 def p4BranchesInGit(branchesAreInRemotes=True):
824 """Find all the branches whose names start with "p4/", looking
825 in remotes or heads as specified by the argument. Return
826 a dictionary of { branch: revision } for each one found.
827 The branch names are the short names, without any
828 "p4/" prefix."""
830 branches = {}
832 cmdline = "git rev-parse --symbolic "
833 if branchesAreInRemotes:
834 cmdline += "--remotes"
835 else:
836 cmdline += "--branches"
838 for line in read_pipe_lines(cmdline):
839 line = line.strip()
841 # only import to p4/
842 if not line.startswith('p4/'):
843 continue
844 # special symbolic ref to p4/master
845 if line == "p4/HEAD":
846 continue
848 # strip off p4/ prefix
849 branch = line[len("p4/"):]
851 branches[branch] = parseRevision(line)
853 return branches
855 def branch_exists(branch):
856 """Make sure that the given ref name really exists."""
858 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
859 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
860 out, _ = p.communicate()
861 if p.returncode:
862 return False
863 # expect exactly one line of output: the branch name
864 return out.rstrip() == branch
866 def findUpstreamBranchPoint(head = "HEAD"):
867 branches = p4BranchesInGit()
868 # map from depot-path to branch name
869 branchByDepotPath = {}
870 for branch in branches.keys():
871 tip = branches[branch]
872 log = extractLogMessageFromGitCommit(tip)
873 settings = extractSettingsGitLog(log)
874 if "depot-paths" in settings:
875 paths = ",".join(settings["depot-paths"])
876 branchByDepotPath[paths] = "remotes/p4/" + branch
878 settings = None
879 parent = 0
880 while parent < 65535:
881 commit = head + "~%s" % parent
882 log = extractLogMessageFromGitCommit(commit)
883 settings = extractSettingsGitLog(log)
884 if "depot-paths" in settings:
885 paths = ",".join(settings["depot-paths"])
886 if paths in branchByDepotPath:
887 return [branchByDepotPath[paths], settings]
889 parent = parent + 1
891 return ["", settings]
893 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
894 if not silent:
895 print("Creating/updating branch(es) in %s based on origin branch(es)"
896 % localRefPrefix)
898 originPrefix = "origin/p4/"
900 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
901 line = line.strip()
902 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
903 continue
905 headName = line[len(originPrefix):]
906 remoteHead = localRefPrefix + headName
907 originHead = line
909 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
910 if ('depot-paths' not in original
911 or 'change' not in original):
912 continue
914 update = False
915 if not gitBranchExists(remoteHead):
916 if verbose:
917 print("creating %s" % remoteHead)
918 update = True
919 else:
920 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
921 if 'change' in settings:
922 if settings['depot-paths'] == original['depot-paths']:
923 originP4Change = int(original['change'])
924 p4Change = int(settings['change'])
925 if originP4Change > p4Change:
926 print("%s (%s) is newer than %s (%s). "
927 "Updating p4 branch from origin."
928 % (originHead, originP4Change,
929 remoteHead, p4Change))
930 update = True
931 else:
932 print("Ignoring: %s was imported from %s while "
933 "%s was imported from %s"
934 % (originHead, ','.join(original['depot-paths']),
935 remoteHead, ','.join(settings['depot-paths'])))
937 if update:
938 system("git update-ref %s %s" % (remoteHead, originHead))
940 def originP4BranchesExist():
941 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
944 def p4ParseNumericChangeRange(parts):
945 changeStart = int(parts[0][1:])
946 if parts[1] == '#head':
947 changeEnd = p4_last_change()
948 else:
949 changeEnd = int(parts[1])
951 return (changeStart, changeEnd)
953 def chooseBlockSize(blockSize):
954 if blockSize:
955 return blockSize
956 else:
957 return defaultBlockSize
959 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
960 assert depotPaths
962 # Parse the change range into start and end. Try to find integer
963 # revision ranges as these can be broken up into blocks to avoid
964 # hitting server-side limits (maxrows, maxscanresults). But if
965 # that doesn't work, fall back to using the raw revision specifier
966 # strings, without using block mode.
968 if changeRange is None or changeRange == '':
969 changeStart = 1
970 changeEnd = p4_last_change()
971 block_size = chooseBlockSize(requestedBlockSize)
972 else:
973 parts = changeRange.split(',')
974 assert len(parts) == 2
975 try:
976 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
977 block_size = chooseBlockSize(requestedBlockSize)
978 except ValueError:
979 changeStart = parts[0][1:]
980 changeEnd = parts[1]
981 if requestedBlockSize:
982 die("cannot use --changes-block-size with non-numeric revisions")
983 block_size = None
985 changes = set()
987 # Retrieve changes a block at a time, to prevent running
988 # into a MaxResults/MaxScanRows error from the server. If
989 # we _do_ hit one of those errors, turn down the block size
991 while True:
992 cmd = ['changes']
994 if block_size:
995 end = min(changeEnd, changeStart + block_size)
996 revisionRange = "%d,%d" % (changeStart, end)
997 else:
998 revisionRange = "%s,%s" % (changeStart, changeEnd)
1000 for p in depotPaths:
1001 cmd += ["%s...@%s" % (p, revisionRange)]
1003 # fetch the changes
1004 try:
1005 result = p4CmdList(cmd, errors_as_exceptions=True)
1006 except P4RequestSizeException as e:
1007 if not block_size:
1008 block_size = e.limit
1009 elif block_size > e.limit:
1010 block_size = e.limit
1011 else:
1012 block_size = max(2, block_size // 2)
1014 if verbose: print("block size error, retrying with block size {0}".format(block_size))
1015 continue
1016 except P4Exception as e:
1017 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1019 # Insert changes in chronological order
1020 for entry in reversed(result):
1021 if 'change' not in entry:
1022 continue
1023 changes.add(int(entry['change']))
1025 if not block_size:
1026 break
1028 if end >= changeEnd:
1029 break
1031 changeStart = end + 1
1033 changes = sorted(changes)
1034 return changes
1036 def p4PathStartsWith(path, prefix):
1037 # This method tries to remedy a potential mixed-case issue:
1039 # If UserA adds //depot/DirA/file1
1040 # and UserB adds //depot/dira/file2
1042 # we may or may not have a problem. If you have core.ignorecase=true,
1043 # we treat DirA and dira as the same directory
1044 if gitConfigBool("core.ignorecase"):
1045 return path.lower().startswith(prefix.lower())
1046 return path.startswith(prefix)
1048 def getClientSpec():
1049 """Look at the p4 client spec, create a View() object that contains
1050 all the mappings, and return it."""
1052 specList = p4CmdList("client -o")
1053 if len(specList) != 1:
1054 die('Output from "client -o" is %d lines, expecting 1' %
1055 len(specList))
1057 # dictionary of all client parameters
1058 entry = specList[0]
1060 # the //client/ name
1061 client_name = entry["Client"]
1063 # just the keys that start with "View"
1064 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1066 # hold this new View
1067 view = View(client_name)
1069 # append the lines, in order, to the view
1070 for view_num in range(len(view_keys)):
1071 k = "View%d" % view_num
1072 if k not in view_keys:
1073 die("Expected view key %s missing" % k)
1074 view.append(entry[k])
1076 return view
1078 def getClientRoot():
1079 """Grab the client directory."""
1081 output = p4CmdList("client -o")
1082 if len(output) != 1:
1083 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1085 entry = output[0]
1086 if "Root" not in entry:
1087 die('Client has no "Root"')
1089 return entry["Root"]
1092 # P4 wildcards are not allowed in filenames. P4 complains
1093 # if you simply add them, but you can force it with "-f", in
1094 # which case it translates them into %xx encoding internally.
1096 def wildcard_decode(path):
1097 # Search for and fix just these four characters. Do % last so
1098 # that fixing it does not inadvertently create new %-escapes.
1099 # Cannot have * in a filename in windows; untested as to
1100 # what p4 would do in such a case.
1101 if not platform.system() == "Windows":
1102 path = path.replace("%2A", "*")
1103 path = path.replace("%23", "#") \
1104 .replace("%40", "@") \
1105 .replace("%25", "%")
1106 return path
1108 def wildcard_encode(path):
1109 # do % first to avoid double-encoding the %s introduced here
1110 path = path.replace("%", "%25") \
1111 .replace("*", "%2A") \
1112 .replace("#", "%23") \
1113 .replace("@", "%40")
1114 return path
1116 def wildcard_present(path):
1117 m = re.search("[*#@%]", path)
1118 return m is not None
1120 class LargeFileSystem(object):
1121 """Base class for large file system support."""
1123 def __init__(self, writeToGitStream):
1124 self.largeFiles = set()
1125 self.writeToGitStream = writeToGitStream
1127 def generatePointer(self, cloneDestination, contentFile):
1128 """Return the content of a pointer file that is stored in Git instead of
1129 the actual content."""
1130 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1132 def pushFile(self, localLargeFile):
1133 """Push the actual content which is not stored in the Git repository to
1134 a server."""
1135 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1137 def hasLargeFileExtension(self, relPath):
1138 return reduce(
1139 lambda a, b: a or b,
1140 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1141 False
1144 def generateTempFile(self, contents):
1145 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1146 for d in contents:
1147 contentFile.write(d)
1148 contentFile.close()
1149 return contentFile.name
1151 def exceedsLargeFileThreshold(self, relPath, contents):
1152 if gitConfigInt('git-p4.largeFileThreshold'):
1153 contentsSize = sum(len(d) for d in contents)
1154 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1155 return True
1156 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1157 contentsSize = sum(len(d) for d in contents)
1158 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159 return False
1160 contentTempFile = self.generateTempFile(contents)
1161 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1162 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1163 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1164 zf.close()
1165 compressedContentsSize = zf.infolist()[0].compress_size
1166 os.remove(contentTempFile)
1167 os.remove(compressedContentFile.name)
1168 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1169 return True
1170 return False
1172 def addLargeFile(self, relPath):
1173 self.largeFiles.add(relPath)
1175 def removeLargeFile(self, relPath):
1176 self.largeFiles.remove(relPath)
1178 def isLargeFile(self, relPath):
1179 return relPath in self.largeFiles
1181 def processContent(self, git_mode, relPath, contents):
1182 """Processes the content of git fast import. This method decides if a
1183 file is stored in the large file system and handles all necessary
1184 steps."""
1185 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1186 contentTempFile = self.generateTempFile(contents)
1187 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1188 if pointer_git_mode:
1189 git_mode = pointer_git_mode
1190 if localLargeFile:
1191 # Move temp file to final location in large file system
1192 largeFileDir = os.path.dirname(localLargeFile)
1193 if not os.path.isdir(largeFileDir):
1194 os.makedirs(largeFileDir)
1195 shutil.move(contentTempFile, localLargeFile)
1196 self.addLargeFile(relPath)
1197 if gitConfigBool('git-p4.largeFilePush'):
1198 self.pushFile(localLargeFile)
1199 if verbose:
1200 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1201 return (git_mode, contents)
1203 class MockLFS(LargeFileSystem):
1204 """Mock large file system for testing."""
1206 def generatePointer(self, contentFile):
1207 """The pointer content is the original content prefixed with "pointer-".
1208 The local filename of the large file storage is derived from the file content.
1210 with open(contentFile, 'r') as f:
1211 content = next(f)
1212 gitMode = '100644'
1213 pointerContents = 'pointer-' + content
1214 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1215 return (gitMode, pointerContents, localLargeFile)
1217 def pushFile(self, localLargeFile):
1218 """The remote filename of the large file storage is the same as the local
1219 one but in a different directory.
1221 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1222 if not os.path.exists(remotePath):
1223 os.makedirs(remotePath)
1224 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1226 class GitLFS(LargeFileSystem):
1227 """Git LFS as backend for the git-p4 large file system.
1228 See https://git-lfs.github.com/ for details."""
1230 def __init__(self, *args):
1231 LargeFileSystem.__init__(self, *args)
1232 self.baseGitAttributes = []
1234 def generatePointer(self, contentFile):
1235 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1236 mode and content which is stored in the Git repository instead of
1237 the actual content. Return also the new location of the actual
1238 content.
1240 if os.path.getsize(contentFile) == 0:
1241 return (None, '', None)
1243 pointerProcess = subprocess.Popen(
1244 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1245 stdout=subprocess.PIPE
1247 pointerFile = pointerProcess.stdout.read()
1248 if pointerProcess.wait():
1249 os.remove(contentFile)
1250 die('git-lfs pointer command failed. Did you install the extension?')
1252 # Git LFS removed the preamble in the output of the 'pointer' command
1253 # starting from version 1.2.0. Check for the preamble here to support
1254 # earlier versions.
1255 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1256 if pointerFile.startswith('Git LFS pointer for'):
1257 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1259 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1260 localLargeFile = os.path.join(
1261 os.getcwd(),
1262 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1263 oid,
1265 # LFS Spec states that pointer files should not have the executable bit set.
1266 gitMode = '100644'
1267 return (gitMode, pointerFile, localLargeFile)
1269 def pushFile(self, localLargeFile):
1270 uploadProcess = subprocess.Popen(
1271 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1273 if uploadProcess.wait():
1274 die('git-lfs push command failed. Did you define a remote?')
1276 def generateGitAttributes(self):
1277 return (
1278 self.baseGitAttributes +
1280 '\n',
1281 '#\n',
1282 '# Git LFS (see https://git-lfs.github.com/)\n',
1283 '#\n',
1285 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1286 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1288 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1289 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1293 def addLargeFile(self, relPath):
1294 LargeFileSystem.addLargeFile(self, relPath)
1295 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1297 def removeLargeFile(self, relPath):
1298 LargeFileSystem.removeLargeFile(self, relPath)
1299 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1301 def processContent(self, git_mode, relPath, contents):
1302 if relPath == '.gitattributes':
1303 self.baseGitAttributes = contents
1304 return (git_mode, self.generateGitAttributes())
1305 else:
1306 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1308 class Command:
1309 delete_actions = ( "delete", "move/delete", "purge" )
1310 add_actions = ( "add", "move/add" )
1312 def __init__(self):
1313 self.usage = "usage: %prog [options]"
1314 self.needsGit = True
1315 self.verbose = False
1317 # This is required for the "append" cloneExclude action
1318 def ensure_value(self, attr, value):
1319 if not hasattr(self, attr) or getattr(self, attr) is None:
1320 setattr(self, attr, value)
1321 return getattr(self, attr)
1323 class P4UserMap:
1324 def __init__(self):
1325 self.userMapFromPerforceServer = False
1326 self.myP4UserId = None
1328 def p4UserId(self):
1329 if self.myP4UserId:
1330 return self.myP4UserId
1332 results = p4CmdList("user -o")
1333 for r in results:
1334 if 'User' in r:
1335 self.myP4UserId = r['User']
1336 return r['User']
1337 die("Could not find your p4 user id")
1339 def p4UserIsMe(self, p4User):
1340 # return True if the given p4 user is actually me
1341 me = self.p4UserId()
1342 if not p4User or p4User != me:
1343 return False
1344 else:
1345 return True
1347 def getUserCacheFilename(self):
1348 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1349 return home + "/.gitp4-usercache.txt"
1351 def getUserMapFromPerforceServer(self):
1352 if self.userMapFromPerforceServer:
1353 return
1354 self.users = {}
1355 self.emails = {}
1357 for output in p4CmdList("users"):
1358 if "User" not in output:
1359 continue
1360 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1361 self.emails[output["Email"]] = output["User"]
1363 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1364 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1365 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1366 if mapUser and len(mapUser[0]) == 3:
1367 user = mapUser[0][0]
1368 fullname = mapUser[0][1]
1369 email = mapUser[0][2]
1370 self.users[user] = fullname + " <" + email + ">"
1371 self.emails[email] = user
1373 s = ''
1374 for (key, val) in self.users.items():
1375 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1377 open(self.getUserCacheFilename(), "wb").write(s)
1378 self.userMapFromPerforceServer = True
1380 def loadUserMapFromCache(self):
1381 self.users = {}
1382 self.userMapFromPerforceServer = False
1383 try:
1384 cache = open(self.getUserCacheFilename(), "rb")
1385 lines = cache.readlines()
1386 cache.close()
1387 for line in lines:
1388 entry = line.strip().split("\t")
1389 self.users[entry[0]] = entry[1]
1390 except IOError:
1391 self.getUserMapFromPerforceServer()
1393 class P4Debug(Command):
1394 def __init__(self):
1395 Command.__init__(self)
1396 self.options = []
1397 self.description = "A tool to debug the output of p4 -G."
1398 self.needsGit = False
1400 def run(self, args):
1401 j = 0
1402 for output in p4CmdList(args):
1403 print('Element: %d' % j)
1404 j += 1
1405 print(output)
1406 return True
1408 class P4RollBack(Command):
1409 def __init__(self):
1410 Command.__init__(self)
1411 self.options = [
1412 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1414 self.description = "A tool to debug the multi-branch import. Don't use :)"
1415 self.rollbackLocalBranches = False
1417 def run(self, args):
1418 if len(args) != 1:
1419 return False
1420 maxChange = int(args[0])
1422 if "p4ExitCode" in p4Cmd("changes -m 1"):
1423 die("Problems executing p4");
1425 if self.rollbackLocalBranches:
1426 refPrefix = "refs/heads/"
1427 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1428 else:
1429 refPrefix = "refs/remotes/"
1430 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1432 for line in lines:
1433 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1434 line = line.strip()
1435 ref = refPrefix + line
1436 log = extractLogMessageFromGitCommit(ref)
1437 settings = extractSettingsGitLog(log)
1439 depotPaths = settings['depot-paths']
1440 change = settings['change']
1442 changed = False
1444 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1445 for p in depotPaths]))) == 0:
1446 print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1447 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1448 continue
1450 while change and int(change) > maxChange:
1451 changed = True
1452 if self.verbose:
1453 print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1454 system("git update-ref %s \"%s^\"" % (ref, ref))
1455 log = extractLogMessageFromGitCommit(ref)
1456 settings = extractSettingsGitLog(log)
1459 depotPaths = settings['depot-paths']
1460 change = settings['change']
1462 if changed:
1463 print("%s rewound to %s" % (ref, change))
1465 return True
1467 class P4Submit(Command, P4UserMap):
1469 conflict_behavior_choices = ("ask", "skip", "quit")
1471 def __init__(self):
1472 Command.__init__(self)
1473 P4UserMap.__init__(self)
1474 self.options = [
1475 optparse.make_option("--origin", dest="origin"),
1476 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1477 # preserve the user, requires relevant p4 permissions
1478 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1479 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1480 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1481 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1482 optparse.make_option("--conflict", dest="conflict_behavior",
1483 choices=self.conflict_behavior_choices),
1484 optparse.make_option("--branch", dest="branch"),
1485 optparse.make_option("--shelve", dest="shelve", action="store_true",
1486 help="Shelve instead of submit. Shelved files are reverted, "
1487 "restoring the workspace to the state before the shelve"),
1488 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1489 metavar="CHANGELIST",
1490 help="update an existing shelved changelist, implies --shelve, "
1491 "repeat in-order for multiple shelved changelists"),
1492 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1493 help="submit only the specified commit(s), one commit or xxx..xxx"),
1494 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1495 help="Disable rebase after submit is completed. Can be useful if you "
1496 "work from a local git branch that is not master"),
1497 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1498 help="Skip Perforce sync of p4/master after submit or shelve"),
1500 self.description = """Submit changes from git to the perforce depot.\n
1501 The `p4-pre-submit` hook is executed if it exists and is executable.
1502 The hook takes no parameters and nothing from standard input. Exiting with
1503 non-zero status from this script prevents `git-p4 submit` from launching.
1505 One usage scenario is to run unit tests in the hook."""
1507 self.usage += " [name of git branch to submit into perforce depot]"
1508 self.origin = ""
1509 self.detectRenames = False
1510 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1511 self.dry_run = False
1512 self.shelve = False
1513 self.update_shelve = list()
1514 self.commit = ""
1515 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1516 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1517 self.prepare_p4_only = False
1518 self.conflict_behavior = None
1519 self.isWindows = (platform.system() == "Windows")
1520 self.exportLabels = False
1521 self.p4HasMoveCommand = p4_has_move_command()
1522 self.branch = None
1524 if gitConfig('git-p4.largeFileSystem'):
1525 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1527 def check(self):
1528 if len(p4CmdList("opened ...")) > 0:
1529 die("You have files opened with perforce! Close them before starting the sync.")
1531 def separate_jobs_from_description(self, message):
1532 """Extract and return a possible Jobs field in the commit
1533 message. It goes into a separate section in the p4 change
1534 specification.
1536 A jobs line starts with "Jobs:" and looks like a new field
1537 in a form. Values are white-space separated on the same
1538 line or on following lines that start with a tab.
1540 This does not parse and extract the full git commit message
1541 like a p4 form. It just sees the Jobs: line as a marker
1542 to pass everything from then on directly into the p4 form,
1543 but outside the description section.
1545 Return a tuple (stripped log message, jobs string)."""
1547 m = re.search(r'^Jobs:', message, re.MULTILINE)
1548 if m is None:
1549 return (message, None)
1551 jobtext = message[m.start():]
1552 stripped_message = message[:m.start()].rstrip()
1553 return (stripped_message, jobtext)
1555 def prepareLogMessage(self, template, message, jobs):
1556 """Edits the template returned from "p4 change -o" to insert
1557 the message in the Description field, and the jobs text in
1558 the Jobs field."""
1559 result = ""
1561 inDescriptionSection = False
1563 for line in template.split("\n"):
1564 if line.startswith("#"):
1565 result += line + "\n"
1566 continue
1568 if inDescriptionSection:
1569 if line.startswith("Files:") or line.startswith("Jobs:"):
1570 inDescriptionSection = False
1571 # insert Jobs section
1572 if jobs:
1573 result += jobs + "\n"
1574 else:
1575 continue
1576 else:
1577 if line.startswith("Description:"):
1578 inDescriptionSection = True
1579 line += "\n"
1580 for messageLine in message.split("\n"):
1581 line += "\t" + messageLine + "\n"
1583 result += line + "\n"
1585 return result
1587 def patchRCSKeywords(self, file, pattern):
1588 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1589 (handle, outFileName) = tempfile.mkstemp(dir='.')
1590 try:
1591 outFile = os.fdopen(handle, "w+")
1592 inFile = open(file, "r")
1593 regexp = re.compile(pattern, re.VERBOSE)
1594 for line in inFile.readlines():
1595 line = regexp.sub(r'$\1$', line)
1596 outFile.write(line)
1597 inFile.close()
1598 outFile.close()
1599 # Forcibly overwrite the original file
1600 os.unlink(file)
1601 shutil.move(outFileName, file)
1602 except:
1603 # cleanup our temporary file
1604 os.unlink(outFileName)
1605 print("Failed to strip RCS keywords in %s" % file)
1606 raise
1608 print("Patched up RCS keywords in %s" % file)
1610 def p4UserForCommit(self,id):
1611 # Return the tuple (perforce user,git email) for a given git commit id
1612 self.getUserMapFromPerforceServer()
1613 gitEmail = read_pipe(["git", "log", "--max-count=1",
1614 "--format=%ae", id])
1615 gitEmail = gitEmail.strip()
1616 if gitEmail not in self.emails:
1617 return (None,gitEmail)
1618 else:
1619 return (self.emails[gitEmail],gitEmail)
1621 def checkValidP4Users(self,commits):
1622 # check if any git authors cannot be mapped to p4 users
1623 for id in commits:
1624 (user,email) = self.p4UserForCommit(id)
1625 if not user:
1626 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1627 if gitConfigBool("git-p4.allowMissingP4Users"):
1628 print("%s" % msg)
1629 else:
1630 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1632 def lastP4Changelist(self):
1633 # Get back the last changelist number submitted in this client spec. This
1634 # then gets used to patch up the username in the change. If the same
1635 # client spec is being used by multiple processes then this might go
1636 # wrong.
1637 results = p4CmdList("client -o") # find the current client
1638 client = None
1639 for r in results:
1640 if 'Client' in r:
1641 client = r['Client']
1642 break
1643 if not client:
1644 die("could not get client spec")
1645 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1646 for r in results:
1647 if 'change' in r:
1648 return r['change']
1649 die("Could not get changelist number for last submit - cannot patch up user details")
1651 def modifyChangelistUser(self, changelist, newUser):
1652 # fixup the user field of a changelist after it has been submitted.
1653 changes = p4CmdList("change -o %s" % changelist)
1654 if len(changes) != 1:
1655 die("Bad output from p4 change modifying %s to user %s" %
1656 (changelist, newUser))
1658 c = changes[0]
1659 if c['User'] == newUser: return # nothing to do
1660 c['User'] = newUser
1661 input = marshal.dumps(c)
1663 result = p4CmdList("change -f -i", stdin=input)
1664 for r in result:
1665 if 'code' in r:
1666 if r['code'] == 'error':
1667 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1668 if 'data' in r:
1669 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1670 return
1671 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1673 def canChangeChangelists(self):
1674 # check to see if we have p4 admin or super-user permissions, either of
1675 # which are required to modify changelists.
1676 results = p4CmdList(["protects", self.depotPath])
1677 for r in results:
1678 if 'perm' in r:
1679 if r['perm'] == 'admin':
1680 return 1
1681 if r['perm'] == 'super':
1682 return 1
1683 return 0
1685 def prepareSubmitTemplate(self, changelist=None):
1686 """Run "p4 change -o" to grab a change specification template.
1687 This does not use "p4 -G", as it is nice to keep the submission
1688 template in original order, since a human might edit it.
1690 Remove lines in the Files section that show changes to files
1691 outside the depot path we're committing into."""
1693 [upstream, settings] = findUpstreamBranchPoint()
1695 template = """\
1696 # A Perforce Change Specification.
1698 # Change: The change number. 'new' on a new changelist.
1699 # Date: The date this specification was last modified.
1700 # Client: The client on which the changelist was created. Read-only.
1701 # User: The user who created the changelist.
1702 # Status: Either 'pending' or 'submitted'. Read-only.
1703 # Type: Either 'public' or 'restricted'. Default is 'public'.
1704 # Description: Comments about the changelist. Required.
1705 # Jobs: What opened jobs are to be closed by this changelist.
1706 # You may delete jobs from this list. (New changelists only.)
1707 # Files: What opened files from the default changelist are to be added
1708 # to this changelist. You may delete files from this list.
1709 # (New changelists only.)
1711 files_list = []
1712 inFilesSection = False
1713 change_entry = None
1714 args = ['change', '-o']
1715 if changelist:
1716 args.append(str(changelist))
1717 for entry in p4CmdList(args):
1718 if 'code' not in entry:
1719 continue
1720 if entry['code'] == 'stat':
1721 change_entry = entry
1722 break
1723 if not change_entry:
1724 die('Failed to decode output of p4 change -o')
1725 for key, value in change_entry.iteritems():
1726 if key.startswith('File'):
1727 if 'depot-paths' in settings:
1728 if not [p for p in settings['depot-paths']
1729 if p4PathStartsWith(value, p)]:
1730 continue
1731 else:
1732 if not p4PathStartsWith(value, self.depotPath):
1733 continue
1734 files_list.append(value)
1735 continue
1736 # Output in the order expected by prepareLogMessage
1737 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1738 if key not in change_entry:
1739 continue
1740 template += '\n'
1741 template += key + ':'
1742 if key == 'Description':
1743 template += '\n'
1744 for field_line in change_entry[key].splitlines():
1745 template += '\t'+field_line+'\n'
1746 if len(files_list) > 0:
1747 template += '\n'
1748 template += 'Files:\n'
1749 for path in files_list:
1750 template += '\t'+path+'\n'
1751 return template
1753 def edit_template(self, template_file):
1754 """Invoke the editor to let the user change the submission
1755 message. Return true if okay to continue with the submit."""
1757 # if configured to skip the editing part, just submit
1758 if gitConfigBool("git-p4.skipSubmitEdit"):
1759 return True
1761 # look at the modification time, to check later if the user saved
1762 # the file
1763 mtime = os.stat(template_file).st_mtime
1765 # invoke the editor
1766 if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1767 editor = os.environ.get("P4EDITOR")
1768 else:
1769 editor = read_pipe("git var GIT_EDITOR").strip()
1770 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1772 # If the file was not saved, prompt to see if this patch should
1773 # be skipped. But skip this verification step if configured so.
1774 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1775 return True
1777 # modification time updated means user saved the file
1778 if os.stat(template_file).st_mtime > mtime:
1779 return True
1781 while True:
1782 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1783 if response == 'y':
1784 return True
1785 if response == 'n':
1786 return False
1788 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1789 # diff
1790 if "P4DIFF" in os.environ:
1791 del(os.environ["P4DIFF"])
1792 diff = ""
1793 for editedFile in editedFiles:
1794 diff += p4_read_pipe(['diff', '-du',
1795 wildcard_encode(editedFile)])
1797 # new file diff
1798 newdiff = ""
1799 for newFile in filesToAdd:
1800 newdiff += "==== new file ====\n"
1801 newdiff += "--- /dev/null\n"
1802 newdiff += "+++ %s\n" % newFile
1804 is_link = os.path.islink(newFile)
1805 expect_link = newFile in symlinks
1807 if is_link and expect_link:
1808 newdiff += "+%s\n" % os.readlink(newFile)
1809 else:
1810 f = open(newFile, "r")
1811 for line in f.readlines():
1812 newdiff += "+" + line
1813 f.close()
1815 return (diff + newdiff).replace('\r\n', '\n')
1817 def applyCommit(self, id):
1818 """Apply one commit, return True if it succeeded."""
1820 print("Applying", read_pipe(["git", "show", "-s",
1821 "--format=format:%h %s", id]))
1823 (p4User, gitEmail) = self.p4UserForCommit(id)
1825 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1826 filesToAdd = set()
1827 filesToChangeType = set()
1828 filesToDelete = set()
1829 editedFiles = set()
1830 pureRenameCopy = set()
1831 symlinks = set()
1832 filesToChangeExecBit = {}
1833 all_files = list()
1835 for line in diff:
1836 diff = parseDiffTreeEntry(line)
1837 modifier = diff['status']
1838 path = diff['src']
1839 all_files.append(path)
1841 if modifier == "M":
1842 p4_edit(path)
1843 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1844 filesToChangeExecBit[path] = diff['dst_mode']
1845 editedFiles.add(path)
1846 elif modifier == "A":
1847 filesToAdd.add(path)
1848 filesToChangeExecBit[path] = diff['dst_mode']
1849 if path in filesToDelete:
1850 filesToDelete.remove(path)
1852 dst_mode = int(diff['dst_mode'], 8)
1853 if dst_mode == 0o120000:
1854 symlinks.add(path)
1856 elif modifier == "D":
1857 filesToDelete.add(path)
1858 if path in filesToAdd:
1859 filesToAdd.remove(path)
1860 elif modifier == "C":
1861 src, dest = diff['src'], diff['dst']
1862 p4_integrate(src, dest)
1863 pureRenameCopy.add(dest)
1864 if diff['src_sha1'] != diff['dst_sha1']:
1865 p4_edit(dest)
1866 pureRenameCopy.discard(dest)
1867 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1868 p4_edit(dest)
1869 pureRenameCopy.discard(dest)
1870 filesToChangeExecBit[dest] = diff['dst_mode']
1871 if self.isWindows:
1872 # turn off read-only attribute
1873 os.chmod(dest, stat.S_IWRITE)
1874 os.unlink(dest)
1875 editedFiles.add(dest)
1876 elif modifier == "R":
1877 src, dest = diff['src'], diff['dst']
1878 if self.p4HasMoveCommand:
1879 p4_edit(src) # src must be open before move
1880 p4_move(src, dest) # opens for (move/delete, move/add)
1881 else:
1882 p4_integrate(src, dest)
1883 if diff['src_sha1'] != diff['dst_sha1']:
1884 p4_edit(dest)
1885 else:
1886 pureRenameCopy.add(dest)
1887 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1888 if not self.p4HasMoveCommand:
1889 p4_edit(dest) # with move: already open, writable
1890 filesToChangeExecBit[dest] = diff['dst_mode']
1891 if not self.p4HasMoveCommand:
1892 if self.isWindows:
1893 os.chmod(dest, stat.S_IWRITE)
1894 os.unlink(dest)
1895 filesToDelete.add(src)
1896 editedFiles.add(dest)
1897 elif modifier == "T":
1898 filesToChangeType.add(path)
1899 else:
1900 die("unknown modifier %s for %s" % (modifier, path))
1902 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1903 patchcmd = diffcmd + " | git apply "
1904 tryPatchCmd = patchcmd + "--check -"
1905 applyPatchCmd = patchcmd + "--check --apply -"
1906 patch_succeeded = True
1908 if os.system(tryPatchCmd) != 0:
1909 fixed_rcs_keywords = False
1910 patch_succeeded = False
1911 print("Unfortunately applying the change failed!")
1913 # Patch failed, maybe it's just RCS keyword woes. Look through
1914 # the patch to see if that's possible.
1915 if gitConfigBool("git-p4.attemptRCSCleanup"):
1916 file = None
1917 pattern = None
1918 kwfiles = {}
1919 for file in editedFiles | filesToDelete:
1920 # did this file's delta contain RCS keywords?
1921 pattern = p4_keywords_regexp_for_file(file)
1923 if pattern:
1924 # this file is a possibility...look for RCS keywords.
1925 regexp = re.compile(pattern, re.VERBOSE)
1926 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1927 if regexp.search(line):
1928 if verbose:
1929 print("got keyword match on %s in %s in %s" % (pattern, line, file))
1930 kwfiles[file] = pattern
1931 break
1933 for file in kwfiles:
1934 if verbose:
1935 print("zapping %s with %s" % (line,pattern))
1936 # File is being deleted, so not open in p4. Must
1937 # disable the read-only bit on windows.
1938 if self.isWindows and file not in editedFiles:
1939 os.chmod(file, stat.S_IWRITE)
1940 self.patchRCSKeywords(file, kwfiles[file])
1941 fixed_rcs_keywords = True
1943 if fixed_rcs_keywords:
1944 print("Retrying the patch with RCS keywords cleaned up")
1945 if os.system(tryPatchCmd) == 0:
1946 patch_succeeded = True
1948 if not patch_succeeded:
1949 for f in editedFiles:
1950 p4_revert(f)
1951 return False
1954 # Apply the patch for real, and do add/delete/+x handling.
1956 system(applyPatchCmd)
1958 for f in filesToChangeType:
1959 p4_edit(f, "-t", "auto")
1960 for f in filesToAdd:
1961 p4_add(f)
1962 for f in filesToDelete:
1963 p4_revert(f)
1964 p4_delete(f)
1966 # Set/clear executable bits
1967 for f in filesToChangeExecBit.keys():
1968 mode = filesToChangeExecBit[f]
1969 setP4ExecBit(f, mode)
1971 update_shelve = 0
1972 if len(self.update_shelve) > 0:
1973 update_shelve = self.update_shelve.pop(0)
1974 p4_reopen_in_change(update_shelve, all_files)
1977 # Build p4 change description, starting with the contents
1978 # of the git commit message.
1980 logMessage = extractLogMessageFromGitCommit(id)
1981 logMessage = logMessage.strip()
1982 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1984 template = self.prepareSubmitTemplate(update_shelve)
1985 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1987 if self.preserveUser:
1988 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1990 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1991 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1992 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1993 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1995 separatorLine = "######## everything below this line is just the diff #######\n"
1996 if not self.prepare_p4_only:
1997 submitTemplate += separatorLine
1998 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2000 (handle, fileName) = tempfile.mkstemp()
2001 tmpFile = os.fdopen(handle, "w+b")
2002 if self.isWindows:
2003 submitTemplate = submitTemplate.replace("\n", "\r\n")
2004 tmpFile.write(submitTemplate)
2005 tmpFile.close()
2007 if self.prepare_p4_only:
2009 # Leave the p4 tree prepared, and the submit template around
2010 # and let the user decide what to do next
2012 print()
2013 print("P4 workspace prepared for submission.")
2014 print("To submit or revert, go to client workspace")
2015 print(" " + self.clientPath)
2016 print()
2017 print("To submit, use \"p4 submit\" to write a new description,")
2018 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2019 " \"git p4\"." % fileName)
2020 print("You can delete the file \"%s\" when finished." % fileName)
2022 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2023 print("To preserve change ownership by user %s, you must\n" \
2024 "do \"p4 change -f <change>\" after submitting and\n" \
2025 "edit the User field.")
2026 if pureRenameCopy:
2027 print("After submitting, renamed files must be re-synced.")
2028 print("Invoke \"p4 sync -f\" on each of these files:")
2029 for f in pureRenameCopy:
2030 print(" " + f)
2032 print()
2033 print("To revert the changes, use \"p4 revert ...\", and delete")
2034 print("the submit template file \"%s\"" % fileName)
2035 if filesToAdd:
2036 print("Since the commit adds new files, they must be deleted:")
2037 for f in filesToAdd:
2038 print(" " + f)
2039 print()
2040 return True
2043 # Let the user edit the change description, then submit it.
2045 submitted = False
2047 try:
2048 if self.edit_template(fileName):
2049 # read the edited message and submit
2050 tmpFile = open(fileName, "rb")
2051 message = tmpFile.read()
2052 tmpFile.close()
2053 if self.isWindows:
2054 message = message.replace("\r\n", "\n")
2055 submitTemplate = message[:message.index(separatorLine)]
2057 if update_shelve:
2058 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2059 elif self.shelve:
2060 p4_write_pipe(['shelve', '-i'], submitTemplate)
2061 else:
2062 p4_write_pipe(['submit', '-i'], submitTemplate)
2063 # The rename/copy happened by applying a patch that created a
2064 # new file. This leaves it writable, which confuses p4.
2065 for f in pureRenameCopy:
2066 p4_sync(f, "-f")
2068 if self.preserveUser:
2069 if p4User:
2070 # Get last changelist number. Cannot easily get it from
2071 # the submit command output as the output is
2072 # unmarshalled.
2073 changelist = self.lastP4Changelist()
2074 self.modifyChangelistUser(changelist, p4User)
2076 submitted = True
2078 finally:
2079 # skip this patch
2080 if not submitted or self.shelve:
2081 if self.shelve:
2082 print ("Reverting shelved files.")
2083 else:
2084 print ("Submission cancelled, undoing p4 changes.")
2085 for f in editedFiles | filesToDelete:
2086 p4_revert(f)
2087 for f in filesToAdd:
2088 p4_revert(f)
2089 os.remove(f)
2091 os.remove(fileName)
2092 return submitted
2094 # Export git tags as p4 labels. Create a p4 label and then tag
2095 # with that.
2096 def exportGitTags(self, gitTags):
2097 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2098 if len(validLabelRegexp) == 0:
2099 validLabelRegexp = defaultLabelRegexp
2100 m = re.compile(validLabelRegexp)
2102 for name in gitTags:
2104 if not m.match(name):
2105 if verbose:
2106 print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2107 continue
2109 # Get the p4 commit this corresponds to
2110 logMessage = extractLogMessageFromGitCommit(name)
2111 values = extractSettingsGitLog(logMessage)
2113 if 'change' not in values:
2114 # a tag pointing to something not sent to p4; ignore
2115 if verbose:
2116 print("git tag %s does not give a p4 commit" % name)
2117 continue
2118 else:
2119 changelist = values['change']
2121 # Get the tag details.
2122 inHeader = True
2123 isAnnotated = False
2124 body = []
2125 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2126 l = l.strip()
2127 if inHeader:
2128 if re.match(r'tag\s+', l):
2129 isAnnotated = True
2130 elif re.match(r'\s*$', l):
2131 inHeader = False
2132 continue
2133 else:
2134 body.append(l)
2136 if not isAnnotated:
2137 body = ["lightweight tag imported by git p4\n"]
2139 # Create the label - use the same view as the client spec we are using
2140 clientSpec = getClientSpec()
2142 labelTemplate = "Label: %s\n" % name
2143 labelTemplate += "Description:\n"
2144 for b in body:
2145 labelTemplate += "\t" + b + "\n"
2146 labelTemplate += "View:\n"
2147 for depot_side in clientSpec.mappings:
2148 labelTemplate += "\t%s\n" % depot_side
2150 if self.dry_run:
2151 print("Would create p4 label %s for tag" % name)
2152 elif self.prepare_p4_only:
2153 print("Not creating p4 label %s for tag due to option" \
2154 " --prepare-p4-only" % name)
2155 else:
2156 p4_write_pipe(["label", "-i"], labelTemplate)
2158 # Use the label
2159 p4_system(["tag", "-l", name] +
2160 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2162 if verbose:
2163 print("created p4 label for tag %s" % name)
2165 def run(self, args):
2166 if len(args) == 0:
2167 self.master = currentGitBranch()
2168 elif len(args) == 1:
2169 self.master = args[0]
2170 if not branchExists(self.master):
2171 die("Branch %s does not exist" % self.master)
2172 else:
2173 return False
2175 for i in self.update_shelve:
2176 if i <= 0:
2177 sys.exit("invalid changelist %d" % i)
2179 if self.master:
2180 allowSubmit = gitConfig("git-p4.allowSubmit")
2181 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2182 die("%s is not in git-p4.allowSubmit" % self.master)
2184 [upstream, settings] = findUpstreamBranchPoint()
2185 self.depotPath = settings['depot-paths'][0]
2186 if len(self.origin) == 0:
2187 self.origin = upstream
2189 if len(self.update_shelve) > 0:
2190 self.shelve = True
2192 if self.preserveUser:
2193 if not self.canChangeChangelists():
2194 die("Cannot preserve user names without p4 super-user or admin permissions")
2196 # if not set from the command line, try the config file
2197 if self.conflict_behavior is None:
2198 val = gitConfig("git-p4.conflict")
2199 if val:
2200 if val not in self.conflict_behavior_choices:
2201 die("Invalid value '%s' for config git-p4.conflict" % val)
2202 else:
2203 val = "ask"
2204 self.conflict_behavior = val
2206 if self.verbose:
2207 print("Origin branch is " + self.origin)
2209 if len(self.depotPath) == 0:
2210 print("Internal error: cannot locate perforce depot path from existing branches")
2211 sys.exit(128)
2213 self.useClientSpec = False
2214 if gitConfigBool("git-p4.useclientspec"):
2215 self.useClientSpec = True
2216 if self.useClientSpec:
2217 self.clientSpecDirs = getClientSpec()
2219 # Check for the existence of P4 branches
2220 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2222 if self.useClientSpec and not branchesDetected:
2223 # all files are relative to the client spec
2224 self.clientPath = getClientRoot()
2225 else:
2226 self.clientPath = p4Where(self.depotPath)
2228 if self.clientPath == "":
2229 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2231 print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2232 self.oldWorkingDirectory = os.getcwd()
2234 # ensure the clientPath exists
2235 new_client_dir = False
2236 if not os.path.exists(self.clientPath):
2237 new_client_dir = True
2238 os.makedirs(self.clientPath)
2240 chdir(self.clientPath, is_client_path=True)
2241 if self.dry_run:
2242 print("Would synchronize p4 checkout in %s" % self.clientPath)
2243 else:
2244 print("Synchronizing p4 checkout...")
2245 if new_client_dir:
2246 # old one was destroyed, and maybe nobody told p4
2247 p4_sync("...", "-f")
2248 else:
2249 p4_sync("...")
2250 self.check()
2252 commits = []
2253 if self.master:
2254 committish = self.master
2255 else:
2256 committish = 'HEAD'
2258 if self.commit != "":
2259 if self.commit.find("..") != -1:
2260 limits_ish = self.commit.split("..")
2261 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2262 commits.append(line.strip())
2263 commits.reverse()
2264 else:
2265 commits.append(self.commit)
2266 else:
2267 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2268 commits.append(line.strip())
2269 commits.reverse()
2271 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2272 self.checkAuthorship = False
2273 else:
2274 self.checkAuthorship = True
2276 if self.preserveUser:
2277 self.checkValidP4Users(commits)
2280 # Build up a set of options to be passed to diff when
2281 # submitting each commit to p4.
2283 if self.detectRenames:
2284 # command-line -M arg
2285 self.diffOpts = "-M"
2286 else:
2287 # If not explicitly set check the config variable
2288 detectRenames = gitConfig("git-p4.detectRenames")
2290 if detectRenames.lower() == "false" or detectRenames == "":
2291 self.diffOpts = ""
2292 elif detectRenames.lower() == "true":
2293 self.diffOpts = "-M"
2294 else:
2295 self.diffOpts = "-M%s" % detectRenames
2297 # no command-line arg for -C or --find-copies-harder, just
2298 # config variables
2299 detectCopies = gitConfig("git-p4.detectCopies")
2300 if detectCopies.lower() == "false" or detectCopies == "":
2301 pass
2302 elif detectCopies.lower() == "true":
2303 self.diffOpts += " -C"
2304 else:
2305 self.diffOpts += " -C%s" % detectCopies
2307 if gitConfigBool("git-p4.detectCopiesHarder"):
2308 self.diffOpts += " --find-copies-harder"
2310 num_shelves = len(self.update_shelve)
2311 if num_shelves > 0 and num_shelves != len(commits):
2312 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2313 (len(commits), num_shelves))
2315 hooks_path = gitConfig("core.hooksPath")
2316 if len(hooks_path) <= 0:
2317 hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2319 hook_file = os.path.join(hooks_path, "p4-pre-submit")
2320 if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2321 sys.exit(1)
2324 # Apply the commits, one at a time. On failure, ask if should
2325 # continue to try the rest of the patches, or quit.
2327 if self.dry_run:
2328 print("Would apply")
2329 applied = []
2330 last = len(commits) - 1
2331 for i, commit in enumerate(commits):
2332 if self.dry_run:
2333 print(" ", read_pipe(["git", "show", "-s",
2334 "--format=format:%h %s", commit]))
2335 ok = True
2336 else:
2337 ok = self.applyCommit(commit)
2338 if ok:
2339 applied.append(commit)
2340 else:
2341 if self.prepare_p4_only and i < last:
2342 print("Processing only the first commit due to option" \
2343 " --prepare-p4-only")
2344 break
2345 if i < last:
2346 quit = False
2347 while True:
2348 # prompt for what to do, or use the option/variable
2349 if self.conflict_behavior == "ask":
2350 print("What do you want to do?")
2351 response = raw_input("[s]kip this commit but apply"
2352 " the rest, or [q]uit? ")
2353 if not response:
2354 continue
2355 elif self.conflict_behavior == "skip":
2356 response = "s"
2357 elif self.conflict_behavior == "quit":
2358 response = "q"
2359 else:
2360 die("Unknown conflict_behavior '%s'" %
2361 self.conflict_behavior)
2363 if response[0] == "s":
2364 print("Skipping this commit, but applying the rest")
2365 break
2366 if response[0] == "q":
2367 print("Quitting")
2368 quit = True
2369 break
2370 if quit:
2371 break
2373 chdir(self.oldWorkingDirectory)
2374 shelved_applied = "shelved" if self.shelve else "applied"
2375 if self.dry_run:
2376 pass
2377 elif self.prepare_p4_only:
2378 pass
2379 elif len(commits) == len(applied):
2380 print("All commits {0}!".format(shelved_applied))
2382 sync = P4Sync()
2383 if self.branch:
2384 sync.branch = self.branch
2385 if self.disable_p4sync:
2386 sync.sync_origin_only()
2387 else:
2388 sync.run([])
2390 if not self.disable_rebase:
2391 rebase = P4Rebase()
2392 rebase.rebase()
2394 else:
2395 if len(applied) == 0:
2396 print("No commits {0}.".format(shelved_applied))
2397 else:
2398 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2399 for c in commits:
2400 if c in applied:
2401 star = "*"
2402 else:
2403 star = " "
2404 print(star, read_pipe(["git", "show", "-s",
2405 "--format=format:%h %s", c]))
2406 print("You will have to do 'git p4 sync' and rebase.")
2408 if gitConfigBool("git-p4.exportLabels"):
2409 self.exportLabels = True
2411 if self.exportLabels:
2412 p4Labels = getP4Labels(self.depotPath)
2413 gitTags = getGitTags()
2415 missingGitTags = gitTags - p4Labels
2416 self.exportGitTags(missingGitTags)
2418 # exit with error unless everything applied perfectly
2419 if len(commits) != len(applied):
2420 sys.exit(1)
2422 return True
2424 class View(object):
2425 """Represent a p4 view ("p4 help views"), and map files in a
2426 repo according to the view."""
2428 def __init__(self, client_name):
2429 self.mappings = []
2430 self.client_prefix = "//%s/" % client_name
2431 # cache results of "p4 where" to lookup client file locations
2432 self.client_spec_path_cache = {}
2434 def append(self, view_line):
2435 """Parse a view line, splitting it into depot and client
2436 sides. Append to self.mappings, preserving order. This
2437 is only needed for tag creation."""
2439 # Split the view line into exactly two words. P4 enforces
2440 # structure on these lines that simplifies this quite a bit.
2442 # Either or both words may be double-quoted.
2443 # Single quotes do not matter.
2444 # Double-quote marks cannot occur inside the words.
2445 # A + or - prefix is also inside the quotes.
2446 # There are no quotes unless they contain a space.
2447 # The line is already white-space stripped.
2448 # The two words are separated by a single space.
2450 if view_line[0] == '"':
2451 # First word is double quoted. Find its end.
2452 close_quote_index = view_line.find('"', 1)
2453 if close_quote_index <= 0:
2454 die("No first-word closing quote found: %s" % view_line)
2455 depot_side = view_line[1:close_quote_index]
2456 # skip closing quote and space
2457 rhs_index = close_quote_index + 1 + 1
2458 else:
2459 space_index = view_line.find(" ")
2460 if space_index <= 0:
2461 die("No word-splitting space found: %s" % view_line)
2462 depot_side = view_line[0:space_index]
2463 rhs_index = space_index + 1
2465 # prefix + means overlay on previous mapping
2466 if depot_side.startswith("+"):
2467 depot_side = depot_side[1:]
2469 # prefix - means exclude this path, leave out of mappings
2470 exclude = False
2471 if depot_side.startswith("-"):
2472 exclude = True
2473 depot_side = depot_side[1:]
2475 if not exclude:
2476 self.mappings.append(depot_side)
2478 def convert_client_path(self, clientFile):
2479 # chop off //client/ part to make it relative
2480 if not clientFile.startswith(self.client_prefix):
2481 die("No prefix '%s' on clientFile '%s'" %
2482 (self.client_prefix, clientFile))
2483 return clientFile[len(self.client_prefix):]
2485 def update_client_spec_path_cache(self, files):
2486 """ Caching file paths by "p4 where" batch query """
2488 # List depot file paths exclude that already cached
2489 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2491 if len(fileArgs) == 0:
2492 return # All files in cache
2494 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2495 for res in where_result:
2496 if "code" in res and res["code"] == "error":
2497 # assume error is "... file(s) not in client view"
2498 continue
2499 if "clientFile" not in res:
2500 die("No clientFile in 'p4 where' output")
2501 if "unmap" in res:
2502 # it will list all of them, but only one not unmap-ped
2503 continue
2504 if gitConfigBool("core.ignorecase"):
2505 res['depotFile'] = res['depotFile'].lower()
2506 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2508 # not found files or unmap files set to ""
2509 for depotFile in fileArgs:
2510 if gitConfigBool("core.ignorecase"):
2511 depotFile = depotFile.lower()
2512 if depotFile not in self.client_spec_path_cache:
2513 self.client_spec_path_cache[depotFile] = ""
2515 def map_in_client(self, depot_path):
2516 """Return the relative location in the client where this
2517 depot file should live. Returns "" if the file should
2518 not be mapped in the client."""
2520 if gitConfigBool("core.ignorecase"):
2521 depot_path = depot_path.lower()
2523 if depot_path in self.client_spec_path_cache:
2524 return self.client_spec_path_cache[depot_path]
2526 die( "Error: %s is not found in client spec path" % depot_path )
2527 return ""
2529 class P4Sync(Command, P4UserMap):
2531 def __init__(self):
2532 Command.__init__(self)
2533 P4UserMap.__init__(self)
2534 self.options = [
2535 optparse.make_option("--branch", dest="branch"),
2536 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2537 optparse.make_option("--changesfile", dest="changesFile"),
2538 optparse.make_option("--silent", dest="silent", action="store_true"),
2539 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2540 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2541 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2542 help="Import into refs/heads/ , not refs/remotes"),
2543 optparse.make_option("--max-changes", dest="maxChanges",
2544 help="Maximum number of changes to import"),
2545 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2546 help="Internal block size to use when iteratively calling p4 changes"),
2547 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2548 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2549 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2550 help="Only sync files that are included in the Perforce Client Spec"),
2551 optparse.make_option("-/", dest="cloneExclude",
2552 action="append", type="string",
2553 help="exclude depot path"),
2555 self.description = """Imports from Perforce into a git repository.\n
2556 example:
2557 //depot/my/project/ -- to import the current head
2558 //depot/my/project/@all -- to import everything
2559 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2561 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2563 self.usage += " //depot/path[@revRange]"
2564 self.silent = False
2565 self.createdBranches = set()
2566 self.committedChanges = set()
2567 self.branch = ""
2568 self.detectBranches = False
2569 self.detectLabels = False
2570 self.importLabels = False
2571 self.changesFile = ""
2572 self.syncWithOrigin = True
2573 self.importIntoRemotes = True
2574 self.maxChanges = ""
2575 self.changes_block_size = None
2576 self.keepRepoPath = False
2577 self.depotPaths = None
2578 self.p4BranchesInGit = []
2579 self.cloneExclude = []
2580 self.useClientSpec = False
2581 self.useClientSpec_from_options = False
2582 self.clientSpecDirs = None
2583 self.tempBranches = []
2584 self.tempBranchLocation = "refs/git-p4-tmp"
2585 self.largeFileSystem = None
2586 self.suppress_meta_comment = False
2588 if gitConfig('git-p4.largeFileSystem'):
2589 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2590 self.largeFileSystem = largeFileSystemConstructor(
2591 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2594 if gitConfig("git-p4.syncFromOrigin") == "false":
2595 self.syncWithOrigin = False
2597 self.depotPaths = []
2598 self.changeRange = ""
2599 self.previousDepotPaths = []
2600 self.hasOrigin = False
2602 # map from branch depot path to parent branch
2603 self.knownBranches = {}
2604 self.initialParents = {}
2606 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2607 self.labels = {}
2609 # Force a checkpoint in fast-import and wait for it to finish
2610 def checkpoint(self):
2611 self.gitStream.write("checkpoint\n\n")
2612 self.gitStream.write("progress checkpoint\n\n")
2613 out = self.gitOutput.readline()
2614 if self.verbose:
2615 print("checkpoint finished: " + out)
2617 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2618 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2619 for path in self.cloneExclude]
2620 files = []
2621 fnum = 0
2622 while "depotFile%s" % fnum in commit:
2623 path = commit["depotFile%s" % fnum]
2625 if [p for p in self.cloneExclude
2626 if p4PathStartsWith(path, p)]:
2627 found = False
2628 else:
2629 found = [p for p in self.depotPaths
2630 if p4PathStartsWith(path, p)]
2631 if not found:
2632 fnum = fnum + 1
2633 continue
2635 file = {}
2636 file["path"] = path
2637 file["rev"] = commit["rev%s" % fnum]
2638 file["action"] = commit["action%s" % fnum]
2639 file["type"] = commit["type%s" % fnum]
2640 if shelved:
2641 file["shelved_cl"] = int(shelved_cl)
2642 files.append(file)
2643 fnum = fnum + 1
2644 return files
2646 def extractJobsFromCommit(self, commit):
2647 jobs = []
2648 jnum = 0
2649 while "job%s" % jnum in commit:
2650 job = commit["job%s" % jnum]
2651 jobs.append(job)
2652 jnum = jnum + 1
2653 return jobs
2655 def stripRepoPath(self, path, prefixes):
2656 """When streaming files, this is called to map a p4 depot path
2657 to where it should go in git. The prefixes are either
2658 self.depotPaths, or self.branchPrefixes in the case of
2659 branch detection."""
2661 if self.useClientSpec:
2662 # branch detection moves files up a level (the branch name)
2663 # from what client spec interpretation gives
2664 path = self.clientSpecDirs.map_in_client(path)
2665 if self.detectBranches:
2666 for b in self.knownBranches:
2667 if path.startswith(b + "/"):
2668 path = path[len(b)+1:]
2670 elif self.keepRepoPath:
2671 # Preserve everything in relative path name except leading
2672 # //depot/; just look at first prefix as they all should
2673 # be in the same depot.
2674 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2675 if p4PathStartsWith(path, depot):
2676 path = path[len(depot):]
2678 else:
2679 for p in prefixes:
2680 if p4PathStartsWith(path, p):
2681 path = path[len(p):]
2682 break
2684 path = wildcard_decode(path)
2685 return path
2687 def splitFilesIntoBranches(self, commit):
2688 """Look at each depotFile in the commit to figure out to what
2689 branch it belongs."""
2691 if self.clientSpecDirs:
2692 files = self.extractFilesFromCommit(commit)
2693 self.clientSpecDirs.update_client_spec_path_cache(files)
2695 branches = {}
2696 fnum = 0
2697 while "depotFile%s" % fnum in commit:
2698 path = commit["depotFile%s" % fnum]
2699 found = [p for p in self.depotPaths
2700 if p4PathStartsWith(path, p)]
2701 if not found:
2702 fnum = fnum + 1
2703 continue
2705 file = {}
2706 file["path"] = path
2707 file["rev"] = commit["rev%s" % fnum]
2708 file["action"] = commit["action%s" % fnum]
2709 file["type"] = commit["type%s" % fnum]
2710 fnum = fnum + 1
2712 # start with the full relative path where this file would
2713 # go in a p4 client
2714 if self.useClientSpec:
2715 relPath = self.clientSpecDirs.map_in_client(path)
2716 else:
2717 relPath = self.stripRepoPath(path, self.depotPaths)
2719 for branch in self.knownBranches.keys():
2720 # add a trailing slash so that a commit into qt/4.2foo
2721 # doesn't end up in qt/4.2, e.g.
2722 if relPath.startswith(branch + "/"):
2723 if branch not in branches:
2724 branches[branch] = []
2725 branches[branch].append(file)
2726 break
2728 return branches
2730 def writeToGitStream(self, gitMode, relPath, contents):
2731 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2732 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2733 for d in contents:
2734 self.gitStream.write(d)
2735 self.gitStream.write('\n')
2737 def encodeWithUTF8(self, path):
2738 try:
2739 path.decode('ascii')
2740 except:
2741 encoding = 'utf8'
2742 if gitConfig('git-p4.pathEncoding'):
2743 encoding = gitConfig('git-p4.pathEncoding')
2744 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2745 if self.verbose:
2746 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2747 return path
2749 # output one file from the P4 stream
2750 # - helper for streamP4Files
2752 def streamOneP4File(self, file, contents):
2753 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2754 relPath = self.encodeWithUTF8(relPath)
2755 if verbose:
2756 if 'fileSize' in self.stream_file:
2757 size = int(self.stream_file['fileSize'])
2758 else:
2759 size = 0 # deleted files don't get a fileSize apparently
2760 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2761 sys.stdout.flush()
2763 (type_base, type_mods) = split_p4_type(file["type"])
2765 git_mode = "100644"
2766 if "x" in type_mods:
2767 git_mode = "100755"
2768 if type_base == "symlink":
2769 git_mode = "120000"
2770 # p4 print on a symlink sometimes contains "target\n";
2771 # if it does, remove the newline
2772 data = ''.join(contents)
2773 if not data:
2774 # Some version of p4 allowed creating a symlink that pointed
2775 # to nothing. This causes p4 errors when checking out such
2776 # a change, and errors here too. Work around it by ignoring
2777 # the bad symlink; hopefully a future change fixes it.
2778 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2779 return
2780 elif data[-1] == '\n':
2781 contents = [data[:-1]]
2782 else:
2783 contents = [data]
2785 if type_base == "utf16":
2786 # p4 delivers different text in the python output to -G
2787 # than it does when using "print -o", or normal p4 client
2788 # operations. utf16 is converted to ascii or utf8, perhaps.
2789 # But ascii text saved as -t utf16 is completely mangled.
2790 # Invoke print -o to get the real contents.
2792 # On windows, the newlines will always be mangled by print, so put
2793 # them back too. This is not needed to the cygwin windows version,
2794 # just the native "NT" type.
2796 try:
2797 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2798 except Exception as e:
2799 if 'Translation of file content failed' in str(e):
2800 type_base = 'binary'
2801 else:
2802 raise e
2803 else:
2804 if p4_version_string().find('/NT') >= 0:
2805 text = text.replace('\r\n', '\n')
2806 contents = [ text ]
2808 if type_base == "apple":
2809 # Apple filetype files will be streamed as a concatenation of
2810 # its appledouble header and the contents. This is useless
2811 # on both macs and non-macs. If using "print -q -o xx", it
2812 # will create "xx" with the data, and "%xx" with the header.
2813 # This is also not very useful.
2815 # Ideally, someday, this script can learn how to generate
2816 # appledouble files directly and import those to git, but
2817 # non-mac machines can never find a use for apple filetype.
2818 print("\nIgnoring apple filetype file %s" % file['depotFile'])
2819 return
2821 # Note that we do not try to de-mangle keywords on utf16 files,
2822 # even though in theory somebody may want that.
2823 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2824 if pattern:
2825 regexp = re.compile(pattern, re.VERBOSE)
2826 text = ''.join(contents)
2827 text = regexp.sub(r'$\1$', text)
2828 contents = [ text ]
2830 if self.largeFileSystem:
2831 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2833 self.writeToGitStream(git_mode, relPath, contents)
2835 def streamOneP4Deletion(self, file):
2836 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2837 relPath = self.encodeWithUTF8(relPath)
2838 if verbose:
2839 sys.stdout.write("delete %s\n" % relPath)
2840 sys.stdout.flush()
2841 self.gitStream.write("D %s\n" % relPath)
2843 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2844 self.largeFileSystem.removeLargeFile(relPath)
2846 # handle another chunk of streaming data
2847 def streamP4FilesCb(self, marshalled):
2849 # catch p4 errors and complain
2850 err = None
2851 if "code" in marshalled:
2852 if marshalled["code"] == "error":
2853 if "data" in marshalled:
2854 err = marshalled["data"].rstrip()
2856 if not err and 'fileSize' in self.stream_file:
2857 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2858 if required_bytes > 0:
2859 err = 'Not enough space left on %s! Free at least %i MB.' % (
2860 os.getcwd(), required_bytes/1024/1024
2863 if err:
2864 f = None
2865 if self.stream_have_file_info:
2866 if "depotFile" in self.stream_file:
2867 f = self.stream_file["depotFile"]
2868 # force a failure in fast-import, else an empty
2869 # commit will be made
2870 self.gitStream.write("\n")
2871 self.gitStream.write("die-now\n")
2872 self.gitStream.close()
2873 # ignore errors, but make sure it exits first
2874 self.importProcess.wait()
2875 if f:
2876 die("Error from p4 print for %s: %s" % (f, err))
2877 else:
2878 die("Error from p4 print: %s" % err)
2880 if 'depotFile' in marshalled and self.stream_have_file_info:
2881 # start of a new file - output the old one first
2882 self.streamOneP4File(self.stream_file, self.stream_contents)
2883 self.stream_file = {}
2884 self.stream_contents = []
2885 self.stream_have_file_info = False
2887 # pick up the new file information... for the
2888 # 'data' field we need to append to our array
2889 for k in marshalled.keys():
2890 if k == 'data':
2891 if 'streamContentSize' not in self.stream_file:
2892 self.stream_file['streamContentSize'] = 0
2893 self.stream_file['streamContentSize'] += len(marshalled['data'])
2894 self.stream_contents.append(marshalled['data'])
2895 else:
2896 self.stream_file[k] = marshalled[k]
2898 if (verbose and
2899 'streamContentSize' in self.stream_file and
2900 'fileSize' in self.stream_file and
2901 'depotFile' in self.stream_file):
2902 size = int(self.stream_file["fileSize"])
2903 if size > 0:
2904 progress = 100*self.stream_file['streamContentSize']/size
2905 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2906 sys.stdout.flush()
2908 self.stream_have_file_info = True
2910 # Stream directly from "p4 files" into "git fast-import"
2911 def streamP4Files(self, files):
2912 filesForCommit = []
2913 filesToRead = []
2914 filesToDelete = []
2916 for f in files:
2917 filesForCommit.append(f)
2918 if f['action'] in self.delete_actions:
2919 filesToDelete.append(f)
2920 else:
2921 filesToRead.append(f)
2923 # deleted files...
2924 for f in filesToDelete:
2925 self.streamOneP4Deletion(f)
2927 if len(filesToRead) > 0:
2928 self.stream_file = {}
2929 self.stream_contents = []
2930 self.stream_have_file_info = False
2932 # curry self argument
2933 def streamP4FilesCbSelf(entry):
2934 self.streamP4FilesCb(entry)
2936 fileArgs = []
2937 for f in filesToRead:
2938 if 'shelved_cl' in f:
2939 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2940 # the contents
2941 fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2942 else:
2943 fileArg = '%s#%s' % (f['path'], f['rev'])
2945 fileArgs.append(fileArg)
2947 p4CmdList(["-x", "-", "print"],
2948 stdin=fileArgs,
2949 cb=streamP4FilesCbSelf)
2951 # do the last chunk
2952 if 'depotFile' in self.stream_file:
2953 self.streamOneP4File(self.stream_file, self.stream_contents)
2955 def make_email(self, userid):
2956 if userid in self.users:
2957 return self.users[userid]
2958 else:
2959 return "%s <a@b>" % userid
2961 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2962 """ Stream a p4 tag.
2963 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2966 if verbose:
2967 print("writing tag %s for commit %s" % (labelName, commit))
2968 gitStream.write("tag %s\n" % labelName)
2969 gitStream.write("from %s\n" % commit)
2971 if 'Owner' in labelDetails:
2972 owner = labelDetails["Owner"]
2973 else:
2974 owner = None
2976 # Try to use the owner of the p4 label, or failing that,
2977 # the current p4 user id.
2978 if owner:
2979 email = self.make_email(owner)
2980 else:
2981 email = self.make_email(self.p4UserId())
2982 tagger = "%s %s %s" % (email, epoch, self.tz)
2984 gitStream.write("tagger %s\n" % tagger)
2986 print("labelDetails=",labelDetails)
2987 if 'Description' in labelDetails:
2988 description = labelDetails['Description']
2989 else:
2990 description = 'Label from git p4'
2992 gitStream.write("data %d\n" % len(description))
2993 gitStream.write(description)
2994 gitStream.write("\n")
2996 def inClientSpec(self, path):
2997 if not self.clientSpecDirs:
2998 return True
2999 inClientSpec = self.clientSpecDirs.map_in_client(path)
3000 if not inClientSpec and self.verbose:
3001 print('Ignoring file outside of client spec: {0}'.format(path))
3002 return inClientSpec
3004 def hasBranchPrefix(self, path):
3005 if not self.branchPrefixes:
3006 return True
3007 hasPrefix = [p for p in self.branchPrefixes
3008 if p4PathStartsWith(path, p)]
3009 if not hasPrefix and self.verbose:
3010 print('Ignoring file outside of prefix: {0}'.format(path))
3011 return hasPrefix
3013 def commit(self, details, files, branch, parent = "", allow_empty=False):
3014 epoch = details["time"]
3015 author = details["user"]
3016 jobs = self.extractJobsFromCommit(details)
3018 if self.verbose:
3019 print('commit into {0}'.format(branch))
3021 if self.clientSpecDirs:
3022 self.clientSpecDirs.update_client_spec_path_cache(files)
3024 files = [f for f in files
3025 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3027 if gitConfigBool('git-p4.keepEmptyCommits'):
3028 allow_empty = True
3030 if not files and not allow_empty:
3031 print('Ignoring revision {0} as it would produce an empty commit.'
3032 .format(details['change']))
3033 return
3035 self.gitStream.write("commit %s\n" % branch)
3036 self.gitStream.write("mark :%s\n" % details["change"])
3037 self.committedChanges.add(int(details["change"]))
3038 committer = ""
3039 if author not in self.users:
3040 self.getUserMapFromPerforceServer()
3041 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3043 self.gitStream.write("committer %s\n" % committer)
3045 self.gitStream.write("data <<EOT\n")
3046 self.gitStream.write(details["desc"])
3047 if len(jobs) > 0:
3048 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3050 if not self.suppress_meta_comment:
3051 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3052 (','.join(self.branchPrefixes), details["change"]))
3053 if len(details['options']) > 0:
3054 self.gitStream.write(": options = %s" % details['options'])
3055 self.gitStream.write("]\n")
3057 self.gitStream.write("EOT\n\n")
3059 if len(parent) > 0:
3060 if self.verbose:
3061 print("parent %s" % parent)
3062 self.gitStream.write("from %s\n" % parent)
3064 self.streamP4Files(files)
3065 self.gitStream.write("\n")
3067 change = int(details["change"])
3069 if change in self.labels:
3070 label = self.labels[change]
3071 labelDetails = label[0]
3072 labelRevisions = label[1]
3073 if self.verbose:
3074 print("Change %s is labelled %s" % (change, labelDetails))
3076 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3077 for p in self.branchPrefixes])
3079 if len(files) == len(labelRevisions):
3081 cleanedFiles = {}
3082 for info in files:
3083 if info["action"] in self.delete_actions:
3084 continue
3085 cleanedFiles[info["depotFile"]] = info["rev"]
3087 if cleanedFiles == labelRevisions:
3088 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3090 else:
3091 if not self.silent:
3092 print("Tag %s does not match with change %s: files do not match."
3093 % (labelDetails["label"], change))
3095 else:
3096 if not self.silent:
3097 print("Tag %s does not match with change %s: file count is different."
3098 % (labelDetails["label"], change))
3100 # Build a dictionary of changelists and labels, for "detect-labels" option.
3101 def getLabels(self):
3102 self.labels = {}
3104 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3105 if len(l) > 0 and not self.silent:
3106 print("Finding files belonging to labels in %s" % self.depotPaths)
3108 for output in l:
3109 label = output["label"]
3110 revisions = {}
3111 newestChange = 0
3112 if self.verbose:
3113 print("Querying files for label %s" % label)
3114 for file in p4CmdList(["files"] +
3115 ["%s...@%s" % (p, label)
3116 for p in self.depotPaths]):
3117 revisions[file["depotFile"]] = file["rev"]
3118 change = int(file["change"])
3119 if change > newestChange:
3120 newestChange = change
3122 self.labels[newestChange] = [output, revisions]
3124 if self.verbose:
3125 print("Label changes: %s" % self.labels.keys())
3127 # Import p4 labels as git tags. A direct mapping does not
3128 # exist, so assume that if all the files are at the same revision
3129 # then we can use that, or it's something more complicated we should
3130 # just ignore.
3131 def importP4Labels(self, stream, p4Labels):
3132 if verbose:
3133 print("import p4 labels: " + ' '.join(p4Labels))
3135 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3136 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3137 if len(validLabelRegexp) == 0:
3138 validLabelRegexp = defaultLabelRegexp
3139 m = re.compile(validLabelRegexp)
3141 for name in p4Labels:
3142 commitFound = False
3144 if not m.match(name):
3145 if verbose:
3146 print("label %s does not match regexp %s" % (name,validLabelRegexp))
3147 continue
3149 if name in ignoredP4Labels:
3150 continue
3152 labelDetails = p4CmdList(['label', "-o", name])[0]
3154 # get the most recent changelist for each file in this label
3155 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3156 for p in self.depotPaths])
3158 if 'change' in change:
3159 # find the corresponding git commit; take the oldest commit
3160 changelist = int(change['change'])
3161 if changelist in self.committedChanges:
3162 gitCommit = ":%d" % changelist # use a fast-import mark
3163 commitFound = True
3164 else:
3165 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3166 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3167 if len(gitCommit) == 0:
3168 print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3169 else:
3170 commitFound = True
3171 gitCommit = gitCommit.strip()
3173 if commitFound:
3174 # Convert from p4 time format
3175 try:
3176 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3177 except ValueError:
3178 print("Could not convert label time %s" % labelDetails['Update'])
3179 tmwhen = 1
3181 when = int(time.mktime(tmwhen))
3182 self.streamTag(stream, name, labelDetails, gitCommit, when)
3183 if verbose:
3184 print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3185 else:
3186 if verbose:
3187 print("Label %s has no changelists - possibly deleted?" % name)
3189 if not commitFound:
3190 # We can't import this label; don't try again as it will get very
3191 # expensive repeatedly fetching all the files for labels that will
3192 # never be imported. If the label is moved in the future, the
3193 # ignore will need to be removed manually.
3194 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3196 def guessProjectName(self):
3197 for p in self.depotPaths:
3198 if p.endswith("/"):
3199 p = p[:-1]
3200 p = p[p.strip().rfind("/") + 1:]
3201 if not p.endswith("/"):
3202 p += "/"
3203 return p
3205 def getBranchMapping(self):
3206 lostAndFoundBranches = set()
3208 user = gitConfig("git-p4.branchUser")
3209 if len(user) > 0:
3210 command = "branches -u %s" % user
3211 else:
3212 command = "branches"
3214 for info in p4CmdList(command):
3215 details = p4Cmd(["branch", "-o", info["branch"]])
3216 viewIdx = 0
3217 while "View%s" % viewIdx in details:
3218 paths = details["View%s" % viewIdx].split(" ")
3219 viewIdx = viewIdx + 1
3220 # require standard //depot/foo/... //depot/bar/... mapping
3221 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3222 continue
3223 source = paths[0]
3224 destination = paths[1]
3225 ## HACK
3226 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3227 source = source[len(self.depotPaths[0]):-4]
3228 destination = destination[len(self.depotPaths[0]):-4]
3230 if destination in self.knownBranches:
3231 if not self.silent:
3232 print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3233 print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3234 continue
3236 self.knownBranches[destination] = source
3238 lostAndFoundBranches.discard(destination)
3240 if source not in self.knownBranches:
3241 lostAndFoundBranches.add(source)
3243 # Perforce does not strictly require branches to be defined, so we also
3244 # check git config for a branch list.
3246 # Example of branch definition in git config file:
3247 # [git-p4]
3248 # branchList=main:branchA
3249 # branchList=main:branchB
3250 # branchList=branchA:branchC
3251 configBranches = gitConfigList("git-p4.branchList")
3252 for branch in configBranches:
3253 if branch:
3254 (source, destination) = branch.split(":")
3255 self.knownBranches[destination] = source
3257 lostAndFoundBranches.discard(destination)
3259 if source not in self.knownBranches:
3260 lostAndFoundBranches.add(source)
3263 for branch in lostAndFoundBranches:
3264 self.knownBranches[branch] = branch
3266 def getBranchMappingFromGitBranches(self):
3267 branches = p4BranchesInGit(self.importIntoRemotes)
3268 for branch in branches.keys():
3269 if branch == "master":
3270 branch = "main"
3271 else:
3272 branch = branch[len(self.projectName):]
3273 self.knownBranches[branch] = branch
3275 def updateOptionDict(self, d):
3276 option_keys = {}
3277 if self.keepRepoPath:
3278 option_keys['keepRepoPath'] = 1
3280 d["options"] = ' '.join(sorted(option_keys.keys()))
3282 def readOptions(self, d):
3283 self.keepRepoPath = ('options' in d
3284 and ('keepRepoPath' in d['options']))
3286 def gitRefForBranch(self, branch):
3287 if branch == "main":
3288 return self.refPrefix + "master"
3290 if len(branch) <= 0:
3291 return branch
3293 return self.refPrefix + self.projectName + branch
3295 def gitCommitByP4Change(self, ref, change):
3296 if self.verbose:
3297 print("looking in ref " + ref + " for change %s using bisect..." % change)
3299 earliestCommit = ""
3300 latestCommit = parseRevision(ref)
3302 while True:
3303 if self.verbose:
3304 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3305 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3306 if len(next) == 0:
3307 if self.verbose:
3308 print("argh")
3309 return ""
3310 log = extractLogMessageFromGitCommit(next)
3311 settings = extractSettingsGitLog(log)
3312 currentChange = int(settings['change'])
3313 if self.verbose:
3314 print("current change %s" % currentChange)
3316 if currentChange == change:
3317 if self.verbose:
3318 print("found %s" % next)
3319 return next
3321 if currentChange < change:
3322 earliestCommit = "^%s" % next
3323 else:
3324 latestCommit = "%s" % next
3326 return ""
3328 def importNewBranch(self, branch, maxChange):
3329 # make fast-import flush all changes to disk and update the refs using the checkpoint
3330 # command so that we can try to find the branch parent in the git history
3331 self.gitStream.write("checkpoint\n\n");
3332 self.gitStream.flush();
3333 branchPrefix = self.depotPaths[0] + branch + "/"
3334 range = "@1,%s" % maxChange
3335 #print "prefix" + branchPrefix
3336 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3337 if len(changes) <= 0:
3338 return False
3339 firstChange = changes[0]
3340 #print "first change in branch: %s" % firstChange
3341 sourceBranch = self.knownBranches[branch]
3342 sourceDepotPath = self.depotPaths[0] + sourceBranch
3343 sourceRef = self.gitRefForBranch(sourceBranch)
3344 #print "source " + sourceBranch
3346 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3347 #print "branch parent: %s" % branchParentChange
3348 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3349 if len(gitParent) > 0:
3350 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3351 #print "parent git commit: %s" % gitParent
3353 self.importChanges(changes)
3354 return True
3356 def searchParent(self, parent, branch, target):
3357 parentFound = False
3358 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3359 "--no-merges", parent]):
3360 blob = blob.strip()
3361 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3362 parentFound = True
3363 if self.verbose:
3364 print("Found parent of %s in commit %s" % (branch, blob))
3365 break
3366 if parentFound:
3367 return blob
3368 else:
3369 return None
3371 def importChanges(self, changes, origin_revision=0):
3372 cnt = 1
3373 for change in changes:
3374 description = p4_describe(change)
3375 self.updateOptionDict(description)
3377 if not self.silent:
3378 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3379 sys.stdout.flush()
3380 cnt = cnt + 1
3382 try:
3383 if self.detectBranches:
3384 branches = self.splitFilesIntoBranches(description)
3385 for branch in branches.keys():
3386 ## HACK --hwn
3387 branchPrefix = self.depotPaths[0] + branch + "/"
3388 self.branchPrefixes = [ branchPrefix ]
3390 parent = ""
3392 filesForCommit = branches[branch]
3394 if self.verbose:
3395 print("branch is %s" % branch)
3397 self.updatedBranches.add(branch)
3399 if branch not in self.createdBranches:
3400 self.createdBranches.add(branch)
3401 parent = self.knownBranches[branch]
3402 if parent == branch:
3403 parent = ""
3404 else:
3405 fullBranch = self.projectName + branch
3406 if fullBranch not in self.p4BranchesInGit:
3407 if not self.silent:
3408 print("\n Importing new branch %s" % fullBranch);
3409 if self.importNewBranch(branch, change - 1):
3410 parent = ""
3411 self.p4BranchesInGit.append(fullBranch)
3412 if not self.silent:
3413 print("\n Resuming with change %s" % change);
3415 if self.verbose:
3416 print("parent determined through known branches: %s" % parent)
3418 branch = self.gitRefForBranch(branch)
3419 parent = self.gitRefForBranch(parent)
3421 if self.verbose:
3422 print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3424 if len(parent) == 0 and branch in self.initialParents:
3425 parent = self.initialParents[branch]
3426 del self.initialParents[branch]
3428 blob = None
3429 if len(parent) > 0:
3430 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3431 if self.verbose:
3432 print("Creating temporary branch: " + tempBranch)
3433 self.commit(description, filesForCommit, tempBranch)
3434 self.tempBranches.append(tempBranch)
3435 self.checkpoint()
3436 blob = self.searchParent(parent, branch, tempBranch)
3437 if blob:
3438 self.commit(description, filesForCommit, branch, blob)
3439 else:
3440 if self.verbose:
3441 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3442 self.commit(description, filesForCommit, branch, parent)
3443 else:
3444 files = self.extractFilesFromCommit(description)
3445 self.commit(description, files, self.branch,
3446 self.initialParent)
3447 # only needed once, to connect to the previous commit
3448 self.initialParent = ""
3449 except IOError:
3450 print(self.gitError.read())
3451 sys.exit(1)
3453 def sync_origin_only(self):
3454 if self.syncWithOrigin:
3455 self.hasOrigin = originP4BranchesExist()
3456 if self.hasOrigin:
3457 if not self.silent:
3458 print('Syncing with origin first, using "git fetch origin"')
3459 system("git fetch origin")
3461 def importHeadRevision(self, revision):
3462 print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3464 details = {}
3465 details["user"] = "git perforce import user"
3466 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3467 % (' '.join(self.depotPaths), revision))
3468 details["change"] = revision
3469 newestRevision = 0
3471 fileCnt = 0
3472 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3474 for info in p4CmdList(["files"] + fileArgs):
3476 if 'code' in info and info['code'] == 'error':
3477 sys.stderr.write("p4 returned an error: %s\n"
3478 % info['data'])
3479 if info['data'].find("must refer to client") >= 0:
3480 sys.stderr.write("This particular p4 error is misleading.\n")
3481 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3482 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3483 sys.exit(1)
3484 if 'p4ExitCode' in info:
3485 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3486 sys.exit(1)
3489 change = int(info["change"])
3490 if change > newestRevision:
3491 newestRevision = change
3493 if info["action"] in self.delete_actions:
3494 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3495 #fileCnt = fileCnt + 1
3496 continue
3498 for prop in ["depotFile", "rev", "action", "type" ]:
3499 details["%s%s" % (prop, fileCnt)] = info[prop]
3501 fileCnt = fileCnt + 1
3503 details["change"] = newestRevision
3505 # Use time from top-most change so that all git p4 clones of
3506 # the same p4 repo have the same commit SHA1s.
3507 res = p4_describe(newestRevision)
3508 details["time"] = res["time"]
3510 self.updateOptionDict(details)
3511 try:
3512 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3513 except IOError:
3514 print("IO error with git fast-import. Is your git version recent enough?")
3515 print(self.gitError.read())
3517 def openStreams(self):
3518 self.importProcess = subprocess.Popen(["git", "fast-import"],
3519 stdin=subprocess.PIPE,
3520 stdout=subprocess.PIPE,
3521 stderr=subprocess.PIPE);
3522 self.gitOutput = self.importProcess.stdout
3523 self.gitStream = self.importProcess.stdin
3524 self.gitError = self.importProcess.stderr
3526 def closeStreams(self):
3527 self.gitStream.close()
3528 if self.importProcess.wait() != 0:
3529 die("fast-import failed: %s" % self.gitError.read())
3530 self.gitOutput.close()
3531 self.gitError.close()
3533 def run(self, args):
3534 if self.importIntoRemotes:
3535 self.refPrefix = "refs/remotes/p4/"
3536 else:
3537 self.refPrefix = "refs/heads/p4/"
3539 self.sync_origin_only()
3541 branch_arg_given = bool(self.branch)
3542 if len(self.branch) == 0:
3543 self.branch = self.refPrefix + "master"
3544 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3545 system("git update-ref %s refs/heads/p4" % self.branch)
3546 system("git branch -D p4")
3548 # accept either the command-line option, or the configuration variable
3549 if self.useClientSpec:
3550 # will use this after clone to set the variable
3551 self.useClientSpec_from_options = True
3552 else:
3553 if gitConfigBool("git-p4.useclientspec"):
3554 self.useClientSpec = True
3555 if self.useClientSpec:
3556 self.clientSpecDirs = getClientSpec()
3558 # TODO: should always look at previous commits,
3559 # merge with previous imports, if possible.
3560 if args == []:
3561 if self.hasOrigin:
3562 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3564 # branches holds mapping from branch name to sha1
3565 branches = p4BranchesInGit(self.importIntoRemotes)
3567 # restrict to just this one, disabling detect-branches
3568 if branch_arg_given:
3569 short = self.branch.split("/")[-1]
3570 if short in branches:
3571 self.p4BranchesInGit = [ short ]
3572 else:
3573 self.p4BranchesInGit = branches.keys()
3575 if len(self.p4BranchesInGit) > 1:
3576 if not self.silent:
3577 print("Importing from/into multiple branches")
3578 self.detectBranches = True
3579 for branch in branches.keys():
3580 self.initialParents[self.refPrefix + branch] = \
3581 branches[branch]
3583 if self.verbose:
3584 print("branches: %s" % self.p4BranchesInGit)
3586 p4Change = 0
3587 for branch in self.p4BranchesInGit:
3588 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3590 settings = extractSettingsGitLog(logMsg)
3592 self.readOptions(settings)
3593 if ('depot-paths' in settings
3594 and 'change' in settings):
3595 change = int(settings['change']) + 1
3596 p4Change = max(p4Change, change)
3598 depotPaths = sorted(settings['depot-paths'])
3599 if self.previousDepotPaths == []:
3600 self.previousDepotPaths = depotPaths
3601 else:
3602 paths = []
3603 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3604 prev_list = prev.split("/")
3605 cur_list = cur.split("/")
3606 for i in range(0, min(len(cur_list), len(prev_list))):
3607 if cur_list[i] != prev_list[i]:
3608 i = i - 1
3609 break
3611 paths.append ("/".join(cur_list[:i + 1]))
3613 self.previousDepotPaths = paths
3615 if p4Change > 0:
3616 self.depotPaths = sorted(self.previousDepotPaths)
3617 self.changeRange = "@%s,#head" % p4Change
3618 if not self.silent and not self.detectBranches:
3619 print("Performing incremental import into %s git branch" % self.branch)
3621 # accept multiple ref name abbreviations:
3622 # refs/foo/bar/branch -> use it exactly
3623 # p4/branch -> prepend refs/remotes/ or refs/heads/
3624 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3625 if not self.branch.startswith("refs/"):
3626 if self.importIntoRemotes:
3627 prepend = "refs/remotes/"
3628 else:
3629 prepend = "refs/heads/"
3630 if not self.branch.startswith("p4/"):
3631 prepend += "p4/"
3632 self.branch = prepend + self.branch
3634 if len(args) == 0 and self.depotPaths:
3635 if not self.silent:
3636 print("Depot paths: %s" % ' '.join(self.depotPaths))
3637 else:
3638 if self.depotPaths and self.depotPaths != args:
3639 print("previous import used depot path %s and now %s was specified. "
3640 "This doesn't work!" % (' '.join (self.depotPaths),
3641 ' '.join (args)))
3642 sys.exit(1)
3644 self.depotPaths = sorted(args)
3646 revision = ""
3647 self.users = {}
3649 # Make sure no revision specifiers are used when --changesfile
3650 # is specified.
3651 bad_changesfile = False
3652 if len(self.changesFile) > 0:
3653 for p in self.depotPaths:
3654 if p.find("@") >= 0 or p.find("#") >= 0:
3655 bad_changesfile = True
3656 break
3657 if bad_changesfile:
3658 die("Option --changesfile is incompatible with revision specifiers")
3660 newPaths = []
3661 for p in self.depotPaths:
3662 if p.find("@") != -1:
3663 atIdx = p.index("@")
3664 self.changeRange = p[atIdx:]
3665 if self.changeRange == "@all":
3666 self.changeRange = ""
3667 elif ',' not in self.changeRange:
3668 revision = self.changeRange
3669 self.changeRange = ""
3670 p = p[:atIdx]
3671 elif p.find("#") != -1:
3672 hashIdx = p.index("#")
3673 revision = p[hashIdx:]
3674 p = p[:hashIdx]
3675 elif self.previousDepotPaths == []:
3676 # pay attention to changesfile, if given, else import
3677 # the entire p4 tree at the head revision
3678 if len(self.changesFile) == 0:
3679 revision = "#head"
3681 p = re.sub ("\.\.\.$", "", p)
3682 if not p.endswith("/"):
3683 p += "/"
3685 newPaths.append(p)
3687 self.depotPaths = newPaths
3689 # --detect-branches may change this for each branch
3690 self.branchPrefixes = self.depotPaths
3692 self.loadUserMapFromCache()
3693 self.labels = {}
3694 if self.detectLabels:
3695 self.getLabels();
3697 if self.detectBranches:
3698 ## FIXME - what's a P4 projectName ?
3699 self.projectName = self.guessProjectName()
3701 if self.hasOrigin:
3702 self.getBranchMappingFromGitBranches()
3703 else:
3704 self.getBranchMapping()
3705 if self.verbose:
3706 print("p4-git branches: %s" % self.p4BranchesInGit)
3707 print("initial parents: %s" % self.initialParents)
3708 for b in self.p4BranchesInGit:
3709 if b != "master":
3711 ## FIXME
3712 b = b[len(self.projectName):]
3713 self.createdBranches.add(b)
3715 self.openStreams()
3717 if revision:
3718 self.importHeadRevision(revision)
3719 else:
3720 changes = []
3722 if len(self.changesFile) > 0:
3723 output = open(self.changesFile).readlines()
3724 changeSet = set()
3725 for line in output:
3726 changeSet.add(int(line))
3728 for change in changeSet:
3729 changes.append(change)
3731 changes.sort()
3732 else:
3733 # catch "git p4 sync" with no new branches, in a repo that
3734 # does not have any existing p4 branches
3735 if len(args) == 0:
3736 if not self.p4BranchesInGit:
3737 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3739 # The default branch is master, unless --branch is used to
3740 # specify something else. Make sure it exists, or complain
3741 # nicely about how to use --branch.
3742 if not self.detectBranches:
3743 if not branch_exists(self.branch):
3744 if branch_arg_given:
3745 die("Error: branch %s does not exist." % self.branch)
3746 else:
3747 die("Error: no branch %s; perhaps specify one with --branch." %
3748 self.branch)
3750 if self.verbose:
3751 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3752 self.changeRange))
3753 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3755 if len(self.maxChanges) > 0:
3756 changes = changes[:min(int(self.maxChanges), len(changes))]
3758 if len(changes) == 0:
3759 if not self.silent:
3760 print("No changes to import!")
3761 else:
3762 if not self.silent and not self.detectBranches:
3763 print("Import destination: %s" % self.branch)
3765 self.updatedBranches = set()
3767 if not self.detectBranches:
3768 if args:
3769 # start a new branch
3770 self.initialParent = ""
3771 else:
3772 # build on a previous revision
3773 self.initialParent = parseRevision(self.branch)
3775 self.importChanges(changes)
3777 if not self.silent:
3778 print("")
3779 if len(self.updatedBranches) > 0:
3780 sys.stdout.write("Updated branches: ")
3781 for b in self.updatedBranches:
3782 sys.stdout.write("%s " % b)
3783 sys.stdout.write("\n")
3785 if gitConfigBool("git-p4.importLabels"):
3786 self.importLabels = True
3788 if self.importLabels:
3789 p4Labels = getP4Labels(self.depotPaths)
3790 gitTags = getGitTags()
3792 missingP4Labels = p4Labels - gitTags
3793 self.importP4Labels(self.gitStream, missingP4Labels)
3795 self.closeStreams()
3797 # Cleanup temporary branches created during import
3798 if self.tempBranches != []:
3799 for branch in self.tempBranches:
3800 read_pipe("git update-ref -d %s" % branch)
3801 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3803 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3804 # a convenient shortcut refname "p4".
3805 if self.importIntoRemotes:
3806 head_ref = self.refPrefix + "HEAD"
3807 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3808 system(["git", "symbolic-ref", head_ref, self.branch])
3810 return True
3812 class P4Rebase(Command):
3813 def __init__(self):
3814 Command.__init__(self)
3815 self.options = [
3816 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3818 self.importLabels = False
3819 self.description = ("Fetches the latest revision from perforce and "
3820 + "rebases the current work (branch) against it")
3822 def run(self, args):
3823 sync = P4Sync()
3824 sync.importLabels = self.importLabels
3825 sync.run([])
3827 return self.rebase()
3829 def rebase(self):
3830 if os.system("git update-index --refresh") != 0:
3831 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3832 if len(read_pipe("git diff-index HEAD --")) > 0:
3833 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3835 [upstream, settings] = findUpstreamBranchPoint()
3836 if len(upstream) == 0:
3837 die("Cannot find upstream branchpoint for rebase")
3839 # the branchpoint may be p4/foo~3, so strip off the parent
3840 upstream = re.sub("~[0-9]+$", "", upstream)
3842 print("Rebasing the current branch onto %s" % upstream)
3843 oldHead = read_pipe("git rev-parse HEAD").strip()
3844 system("git rebase %s" % upstream)
3845 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3846 return True
3848 class P4Clone(P4Sync):
3849 def __init__(self):
3850 P4Sync.__init__(self)
3851 self.description = "Creates a new git repository and imports from Perforce into it"
3852 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3853 self.options += [
3854 optparse.make_option("--destination", dest="cloneDestination",
3855 action='store', default=None,
3856 help="where to leave result of the clone"),
3857 optparse.make_option("--bare", dest="cloneBare",
3858 action="store_true", default=False),
3860 self.cloneDestination = None
3861 self.needsGit = False
3862 self.cloneBare = False
3864 def defaultDestination(self, args):
3865 ## TODO: use common prefix of args?
3866 depotPath = args[0]
3867 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3868 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3869 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3870 depotDir = re.sub(r"/$", "", depotDir)
3871 return os.path.split(depotDir)[1]
3873 def run(self, args):
3874 if len(args) < 1:
3875 return False
3877 if self.keepRepoPath and not self.cloneDestination:
3878 sys.stderr.write("Must specify destination for --keep-path\n")
3879 sys.exit(1)
3881 depotPaths = args
3883 if not self.cloneDestination and len(depotPaths) > 1:
3884 self.cloneDestination = depotPaths[-1]
3885 depotPaths = depotPaths[:-1]
3887 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3888 for p in depotPaths:
3889 if not p.startswith("//"):
3890 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3891 return False
3893 if not self.cloneDestination:
3894 self.cloneDestination = self.defaultDestination(args)
3896 print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3898 if not os.path.exists(self.cloneDestination):
3899 os.makedirs(self.cloneDestination)
3900 chdir(self.cloneDestination)
3902 init_cmd = [ "git", "init" ]
3903 if self.cloneBare:
3904 init_cmd.append("--bare")
3905 retcode = subprocess.call(init_cmd)
3906 if retcode:
3907 raise CalledProcessError(retcode, init_cmd)
3909 if not P4Sync.run(self, depotPaths):
3910 return False
3912 # create a master branch and check out a work tree
3913 if gitBranchExists(self.branch):
3914 system([ "git", "branch", "master", self.branch ])
3915 if not self.cloneBare:
3916 system([ "git", "checkout", "-f" ])
3917 else:
3918 print('Not checking out any branch, use ' \
3919 '"git checkout -q -b master <branch>"')
3921 # auto-set this variable if invoked with --use-client-spec
3922 if self.useClientSpec_from_options:
3923 system("git config --bool git-p4.useclientspec true")
3925 return True
3927 class P4Unshelve(Command):
3928 def __init__(self):
3929 Command.__init__(self)
3930 self.options = []
3931 self.origin = "HEAD"
3932 self.description = "Unshelve a P4 changelist into a git commit"
3933 self.usage = "usage: %prog [options] changelist"
3934 self.options += [
3935 optparse.make_option("--origin", dest="origin",
3936 help="Use this base revision instead of the default (%s)" % self.origin),
3938 self.verbose = False
3939 self.noCommit = False
3940 self.destbranch = "refs/remotes/p4-unshelved"
3942 def renameBranch(self, branch_name):
3943 """ Rename the existing branch to branch_name.N
3946 found = True
3947 for i in range(0,1000):
3948 backup_branch_name = "{0}.{1}".format(branch_name, i)
3949 if not gitBranchExists(backup_branch_name):
3950 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3951 gitDeleteRef(branch_name)
3952 found = True
3953 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3954 break
3956 if not found:
3957 sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3959 def findLastP4Revision(self, starting_point):
3960 """ Look back from starting_point for the first commit created by git-p4
3961 to find the P4 commit we are based on, and the depot-paths.
3964 for parent in (range(65535)):
3965 log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3966 settings = extractSettingsGitLog(log)
3967 if 'change' in settings:
3968 return settings
3970 sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3972 def createShelveParent(self, change, branch_name, sync, origin):
3973 """ Create a commit matching the parent of the shelved changelist 'change'
3975 parent_description = p4_describe(change, shelved=True)
3976 parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
3977 files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
3979 parent_files = []
3980 for f in files:
3981 # if it was added in the shelved changelist, it won't exist in the parent
3982 if f['action'] in self.add_actions:
3983 continue
3985 # if it was deleted in the shelved changelist it must not be deleted
3986 # in the parent - we might even need to create it if the origin branch
3987 # does not have it
3988 if f['action'] in self.delete_actions:
3989 f['action'] = 'add'
3991 parent_files.append(f)
3993 sync.commit(parent_description, parent_files, branch_name,
3994 parent=origin, allow_empty=True)
3995 print("created parent commit for {0} based on {1} in {2}".format(
3996 change, self.origin, branch_name))
3998 def run(self, args):
3999 if len(args) != 1:
4000 return False
4002 if not gitBranchExists(self.origin):
4003 sys.exit("origin branch {0} does not exist".format(self.origin))
4005 sync = P4Sync()
4006 changes = args
4008 # only one change at a time
4009 change = changes[0]
4011 # if the target branch already exists, rename it
4012 branch_name = "{0}/{1}".format(self.destbranch, change)
4013 if gitBranchExists(branch_name):
4014 self.renameBranch(branch_name)
4015 sync.branch = branch_name
4017 sync.verbose = self.verbose
4018 sync.suppress_meta_comment = True
4020 settings = self.findLastP4Revision(self.origin)
4021 sync.depotPaths = settings['depot-paths']
4022 sync.branchPrefixes = sync.depotPaths
4024 sync.openStreams()
4025 sync.loadUserMapFromCache()
4026 sync.silent = True
4028 # create a commit for the parent of the shelved changelist
4029 self.createShelveParent(change, branch_name, sync, self.origin)
4031 # create the commit for the shelved changelist itself
4032 description = p4_describe(change, True)
4033 files = sync.extractFilesFromCommit(description, True, change)
4035 sync.commit(description, files, branch_name, "")
4036 sync.closeStreams()
4038 print("unshelved changelist {0} into {1}".format(change, branch_name))
4040 return True
4042 class P4Branches(Command):
4043 def __init__(self):
4044 Command.__init__(self)
4045 self.options = [ ]
4046 self.description = ("Shows the git branches that hold imports and their "
4047 + "corresponding perforce depot paths")
4048 self.verbose = False
4050 def run(self, args):
4051 if originP4BranchesExist():
4052 createOrUpdateBranchesFromOrigin()
4054 cmdline = "git rev-parse --symbolic "
4055 cmdline += " --remotes"
4057 for line in read_pipe_lines(cmdline):
4058 line = line.strip()
4060 if not line.startswith('p4/') or line == "p4/HEAD":
4061 continue
4062 branch = line
4064 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4065 settings = extractSettingsGitLog(log)
4067 print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4068 return True
4070 class HelpFormatter(optparse.IndentedHelpFormatter):
4071 def __init__(self):
4072 optparse.IndentedHelpFormatter.__init__(self)
4074 def format_description(self, description):
4075 if description:
4076 return description + "\n"
4077 else:
4078 return ""
4080 def printUsage(commands):
4081 print("usage: %s <command> [options]" % sys.argv[0])
4082 print("")
4083 print("valid commands: %s" % ", ".join(commands))
4084 print("")
4085 print("Try %s <command> --help for command specific help." % sys.argv[0])
4086 print("")
4088 commands = {
4089 "debug" : P4Debug,
4090 "submit" : P4Submit,
4091 "commit" : P4Submit,
4092 "sync" : P4Sync,
4093 "rebase" : P4Rebase,
4094 "clone" : P4Clone,
4095 "rollback" : P4RollBack,
4096 "branches" : P4Branches,
4097 "unshelve" : P4Unshelve,
4101 def main():
4102 if len(sys.argv[1:]) == 0:
4103 printUsage(commands.keys())
4104 sys.exit(2)
4106 cmdName = sys.argv[1]
4107 try:
4108 klass = commands[cmdName]
4109 cmd = klass()
4110 except KeyError:
4111 print("unknown command %s" % cmdName)
4112 print("")
4113 printUsage(commands.keys())
4114 sys.exit(2)
4116 options = cmd.options
4117 cmd.gitdir = os.environ.get("GIT_DIR", None)
4119 args = sys.argv[2:]
4121 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4122 if cmd.needsGit:
4123 options.append(optparse.make_option("--git-dir", dest="gitdir"))
4125 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4126 options,
4127 description = cmd.description,
4128 formatter = HelpFormatter())
4130 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4131 global verbose
4132 verbose = cmd.verbose
4133 if cmd.needsGit:
4134 if cmd.gitdir == None:
4135 cmd.gitdir = os.path.abspath(".git")
4136 if not isValidGitDir(cmd.gitdir):
4137 # "rev-parse --git-dir" without arguments will try $PWD/.git
4138 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4139 if os.path.exists(cmd.gitdir):
4140 cdup = read_pipe("git rev-parse --show-cdup").strip()
4141 if len(cdup) > 0:
4142 chdir(cdup);
4144 if not isValidGitDir(cmd.gitdir):
4145 if isValidGitDir(cmd.gitdir + "/.git"):
4146 cmd.gitdir += "/.git"
4147 else:
4148 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4150 # so git commands invoked from the P4 workspace will succeed
4151 os.environ["GIT_DIR"] = cmd.gitdir
4153 if not cmd.run(args):
4154 parser.print_help()
4155 sys.exit(2)
4158 if __name__ == '__main__':
4159 main()