generate-cmdlist.sh: collect config from all config.txt files
[git/debian.git] / git-p4.py
blob7fab255584e5bca70aef305f6ead73330694a88b
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
30 # support basestring in python3
31 try:
32 unicode = unicode
33 except NameError:
34 # 'unicode' is undefined, must be Python 3
35 str = str
36 unicode = str
37 bytes = bytes
38 basestring = (str,bytes)
39 else:
40 # 'unicode' exists, must be Python 2
41 str = str
42 unicode = unicode
43 bytes = str
44 basestring = basestring
46 try:
47 from subprocess import CalledProcessError
48 except ImportError:
49 # from python2.7:subprocess.py
50 # Exception classes used by this module.
51 class CalledProcessError(Exception):
52 """This exception is raised when a process run by check_call() returns
53 a non-zero exit status. The exit status will be stored in the
54 returncode attribute."""
55 def __init__(self, returncode, cmd):
56 self.returncode = returncode
57 self.cmd = cmd
58 def __str__(self):
59 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
61 verbose = False
63 # Only labels/tags matching this will be imported/exported
64 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
66 # The block size is reduced automatically if required
67 defaultBlockSize = 1<<20
69 p4_access_checked = False
71 def p4_build_cmd(cmd):
72 """Build a suitable p4 command line.
74 This consolidates building and returning a p4 command line into one
75 location. It means that hooking into the environment, or other configuration
76 can be done more easily.
77 """
78 real_cmd = ["p4"]
80 user = gitConfig("git-p4.user")
81 if len(user) > 0:
82 real_cmd += ["-u",user]
84 password = gitConfig("git-p4.password")
85 if len(password) > 0:
86 real_cmd += ["-P", password]
88 port = gitConfig("git-p4.port")
89 if len(port) > 0:
90 real_cmd += ["-p", port]
92 host = gitConfig("git-p4.host")
93 if len(host) > 0:
94 real_cmd += ["-H", host]
96 client = gitConfig("git-p4.client")
97 if len(client) > 0:
98 real_cmd += ["-c", client]
100 retries = gitConfigInt("git-p4.retries")
101 if retries is None:
102 # Perform 3 retries by default
103 retries = 3
104 if retries > 0:
105 # Provide a way to not pass this option by setting git-p4.retries to 0
106 real_cmd += ["-r", str(retries)]
108 if isinstance(cmd,basestring):
109 real_cmd = ' '.join(real_cmd) + ' ' + cmd
110 else:
111 real_cmd += cmd
113 # now check that we can actually talk to the server
114 global p4_access_checked
115 if not p4_access_checked:
116 p4_access_checked = True # suppress access checks in p4_check_access itself
117 p4_check_access()
119 return real_cmd
121 def git_dir(path):
122 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123 This won't automatically add ".git" to a directory.
125 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126 if not d or len(d) == 0:
127 return None
128 else:
129 return d
131 def chdir(path, is_client_path=False):
132 """Do chdir to the given path, and set the PWD environment
133 variable for use by P4. It does not look at getcwd() output.
134 Since we're not using the shell, it is necessary to set the
135 PWD environment variable explicitly.
137 Normally, expand the path to force it to be absolute. This
138 addresses the use of relative path names inside P4 settings,
139 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
140 as given; it looks for .p4config using PWD.
142 If is_client_path, the path was handed to us directly by p4,
143 and may be a symbolic link. Do not call os.getcwd() in this
144 case, because it will cause p4 to think that PWD is not inside
145 the client path.
148 os.chdir(path)
149 if not is_client_path:
150 path = os.getcwd()
151 os.environ['PWD'] = path
153 def calcDiskFree():
154 """Return free space in bytes on the disk of the given dirname."""
155 if platform.system() == 'Windows':
156 free_bytes = ctypes.c_ulonglong(0)
157 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158 return free_bytes.value
159 else:
160 st = os.statvfs(os.getcwd())
161 return st.f_bavail * st.f_frsize
163 def die(msg):
164 if verbose:
165 raise Exception(msg)
166 else:
167 sys.stderr.write(msg + "\n")
168 sys.exit(1)
170 def write_pipe(c, stdin):
171 if verbose:
172 sys.stderr.write('Writing pipe: %s\n' % str(c))
174 expand = isinstance(c,basestring)
175 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176 pipe = p.stdin
177 val = pipe.write(stdin)
178 pipe.close()
179 if p.wait():
180 die('Command failed: %s' % str(c))
182 return val
184 def p4_write_pipe(c, stdin):
185 real_cmd = p4_build_cmd(c)
186 return write_pipe(real_cmd, stdin)
188 def read_pipe_full(c):
189 """ Read output from command. Returns a tuple
190 of the return status, stdout text and stderr
191 text.
193 if verbose:
194 sys.stderr.write('Reading pipe: %s\n' % str(c))
196 expand = isinstance(c,basestring)
197 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198 (out, err) = p.communicate()
199 return (p.returncode, out, err)
201 def read_pipe(c, ignore_error=False):
202 """ Read output from command. Returns the output text on
203 success. On failure, terminates execution, unless
204 ignore_error is True, when it returns an empty string.
206 (retcode, out, err) = read_pipe_full(c)
207 if retcode != 0:
208 if ignore_error:
209 out = ""
210 else:
211 die('Command failed: %s\nError: %s' % (str(c), err))
212 return out
214 def read_pipe_text(c):
215 """ Read output from a command with trailing whitespace stripped.
216 On error, returns None.
218 (retcode, out, err) = read_pipe_full(c)
219 if retcode != 0:
220 return None
221 else:
222 return out.rstrip()
224 def p4_read_pipe(c, ignore_error=False):
225 real_cmd = p4_build_cmd(c)
226 return read_pipe(real_cmd, ignore_error)
228 def read_pipe_lines(c):
229 if verbose:
230 sys.stderr.write('Reading pipe: %s\n' % str(c))
232 expand = isinstance(c, basestring)
233 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234 pipe = p.stdout
235 val = pipe.readlines()
236 if pipe.close() or p.wait():
237 die('Command failed: %s' % str(c))
239 return val
241 def p4_read_pipe_lines(c):
242 """Specifically invoke p4 on the command supplied. """
243 real_cmd = p4_build_cmd(c)
244 return read_pipe_lines(real_cmd)
246 def p4_has_command(cmd):
247 """Ask p4 for help on this command. If it returns an error, the
248 command does not exist in this version of p4."""
249 real_cmd = p4_build_cmd(["help", cmd])
250 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251 stderr=subprocess.PIPE)
252 p.communicate()
253 return p.returncode == 0
255 def p4_has_move_command():
256 """See if the move command exists, that it supports -k, and that
257 it has not been administratively disabled. The arguments
258 must be correct, but the filenames do not have to exist. Use
259 ones with wildcards so even if they exist, it will fail."""
261 if not p4_has_command("move"):
262 return False
263 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265 (out, err) = p.communicate()
266 # return code will be 1 in either case
267 if err.find("Invalid option") >= 0:
268 return False
269 if err.find("disabled") >= 0:
270 return False
271 # assume it failed because @... was invalid changelist
272 return True
274 def system(cmd, ignore_error=False):
275 expand = isinstance(cmd,basestring)
276 if verbose:
277 sys.stderr.write("executing %s\n" % str(cmd))
278 retcode = subprocess.call(cmd, shell=expand)
279 if retcode and not ignore_error:
280 raise CalledProcessError(retcode, cmd)
282 return retcode
284 def p4_system(cmd):
285 """Specifically invoke p4 as the system command. """
286 real_cmd = p4_build_cmd(cmd)
287 expand = isinstance(real_cmd, basestring)
288 retcode = subprocess.call(real_cmd, shell=expand)
289 if retcode:
290 raise CalledProcessError(retcode, real_cmd)
292 def die_bad_access(s):
293 die("failure accessing depot: {0}".format(s.rstrip()))
295 def p4_check_access(min_expiration=1):
296 """ Check if we can access Perforce - account still logged in
298 results = p4CmdList(["login", "-s"])
300 if len(results) == 0:
301 # should never get here: always get either some results, or a p4ExitCode
302 assert("could not parse response from perforce")
304 result = results[0]
306 if 'p4ExitCode' in result:
307 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308 die_bad_access("could not run p4")
310 code = result.get("code")
311 if not code:
312 # we get here if we couldn't connect and there was nothing to unmarshal
313 die_bad_access("could not connect")
315 elif code == "stat":
316 expiry = result.get("TicketExpiration")
317 if expiry:
318 expiry = int(expiry)
319 if expiry > min_expiration:
320 # ok to carry on
321 return
322 else:
323 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
325 else:
326 # account without a timeout - all ok
327 return
329 elif code == "error":
330 data = result.get("data")
331 if data:
332 die_bad_access("p4 error: {0}".format(data))
333 else:
334 die_bad_access("unknown error")
335 else:
336 die_bad_access("unknown error code {0}".format(code))
338 _p4_version_string = None
339 def p4_version_string():
340 """Read the version string, showing just the last line, which
341 hopefully is the interesting version bit.
343 $ p4 -V
344 Perforce - The Fast Software Configuration Management System.
345 Copyright 1995-2011 Perforce Software. All rights reserved.
346 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
348 global _p4_version_string
349 if not _p4_version_string:
350 a = p4_read_pipe_lines(["-V"])
351 _p4_version_string = a[-1].rstrip()
352 return _p4_version_string
354 def p4_integrate(src, dest):
355 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
357 def p4_sync(f, *options):
358 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
360 def p4_add(f):
361 # forcibly add file names with wildcards
362 if wildcard_present(f):
363 p4_system(["add", "-f", f])
364 else:
365 p4_system(["add", f])
367 def p4_delete(f):
368 p4_system(["delete", wildcard_encode(f)])
370 def p4_edit(f, *options):
371 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
373 def p4_revert(f):
374 p4_system(["revert", wildcard_encode(f)])
376 def p4_reopen(type, f):
377 p4_system(["reopen", "-t", type, wildcard_encode(f)])
379 def p4_reopen_in_change(changelist, files):
380 cmd = ["reopen", "-c", str(changelist)] + files
381 p4_system(cmd)
383 def p4_move(src, dest):
384 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
386 def p4_last_change():
387 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
388 return int(results[0]['change'])
390 def p4_describe(change, shelved=False):
391 """Make sure it returns a valid result by checking for
392 the presence of field "time". Return a dict of the
393 results."""
395 cmd = ["describe", "-s"]
396 if shelved:
397 cmd += ["-S"]
398 cmd += [str(change)]
400 ds = p4CmdList(cmd, skip_info=True)
401 if len(ds) != 1:
402 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
404 d = ds[0]
406 if "p4ExitCode" in d:
407 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
408 str(d)))
409 if "code" in d:
410 if d["code"] == "error":
411 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
413 if "time" not in d:
414 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
416 return d
419 # Canonicalize the p4 type and return a tuple of the
420 # base type, plus any modifiers. See "p4 help filetypes"
421 # for a list and explanation.
423 def split_p4_type(p4type):
425 p4_filetypes_historical = {
426 "ctempobj": "binary+Sw",
427 "ctext": "text+C",
428 "cxtext": "text+Cx",
429 "ktext": "text+k",
430 "kxtext": "text+kx",
431 "ltext": "text+F",
432 "tempobj": "binary+FSw",
433 "ubinary": "binary+F",
434 "uresource": "resource+F",
435 "uxbinary": "binary+Fx",
436 "xbinary": "binary+x",
437 "xltext": "text+Fx",
438 "xtempobj": "binary+Swx",
439 "xtext": "text+x",
440 "xunicode": "unicode+x",
441 "xutf16": "utf16+x",
443 if p4type in p4_filetypes_historical:
444 p4type = p4_filetypes_historical[p4type]
445 mods = ""
446 s = p4type.split("+")
447 base = s[0]
448 mods = ""
449 if len(s) > 1:
450 mods = s[1]
451 return (base, mods)
454 # return the raw p4 type of a file (text, text+ko, etc)
456 def p4_type(f):
457 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
458 return results[0]['headType']
461 # Given a type base and modifier, return a regexp matching
462 # the keywords that can be expanded in the file
464 def p4_keywords_regexp_for_type(base, type_mods):
465 if base in ("text", "unicode", "binary"):
466 kwords = None
467 if "ko" in type_mods:
468 kwords = 'Id|Header'
469 elif "k" in type_mods:
470 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
471 else:
472 return None
473 pattern = r"""
474 \$ # Starts with a dollar, followed by...
475 (%s) # one of the keywords, followed by...
476 (:[^$\n]+)? # possibly an old expansion, followed by...
477 \$ # another dollar
478 """ % kwords
479 return pattern
480 else:
481 return None
484 # Given a file, return a regexp matching the possible
485 # RCS keywords that will be expanded, or None for files
486 # with kw expansion turned off.
488 def p4_keywords_regexp_for_file(file):
489 if not os.path.exists(file):
490 return None
491 else:
492 (type_base, type_mods) = split_p4_type(p4_type(file))
493 return p4_keywords_regexp_for_type(type_base, type_mods)
495 def setP4ExecBit(file, mode):
496 # Reopens an already open file and changes the execute bit to match
497 # the execute bit setting in the passed in mode.
499 p4Type = "+x"
501 if not isModeExec(mode):
502 p4Type = getP4OpenedType(file)
503 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
504 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
505 if p4Type[-1] == "+":
506 p4Type = p4Type[0:-1]
508 p4_reopen(p4Type, file)
510 def getP4OpenedType(file):
511 # Returns the perforce file type for the given file.
513 result = p4_read_pipe(["opened", wildcard_encode(file)])
514 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
515 if match:
516 return match.group(1)
517 else:
518 die("Could not determine file type for %s (result: '%s')" % (file, result))
520 # Return the set of all p4 labels
521 def getP4Labels(depotPaths):
522 labels = set()
523 if isinstance(depotPaths,basestring):
524 depotPaths = [depotPaths]
526 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
527 label = l['label']
528 labels.add(label)
530 return labels
532 # Return the set of all git tags
533 def getGitTags():
534 gitTags = set()
535 for line in read_pipe_lines(["git", "tag"]):
536 tag = line.strip()
537 gitTags.add(tag)
538 return gitTags
540 def diffTreePattern():
541 # This is a simple generator for the diff tree regex pattern. This could be
542 # a class variable if this and parseDiffTreeEntry were a part of a class.
543 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
544 while True:
545 yield pattern
547 def parseDiffTreeEntry(entry):
548 """Parses a single diff tree entry into its component elements.
550 See git-diff-tree(1) manpage for details about the format of the diff
551 output. This method returns a dictionary with the following elements:
553 src_mode - The mode of the source file
554 dst_mode - The mode of the destination file
555 src_sha1 - The sha1 for the source file
556 dst_sha1 - The sha1 fr the destination file
557 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
558 status_score - The score for the status (applicable for 'C' and 'R'
559 statuses). This is None if there is no score.
560 src - The path for the source file.
561 dst - The path for the destination file. This is only present for
562 copy or renames. If it is not present, this is None.
564 If the pattern is not matched, None is returned."""
566 match = diffTreePattern().next().match(entry)
567 if match:
568 return {
569 'src_mode': match.group(1),
570 'dst_mode': match.group(2),
571 'src_sha1': match.group(3),
572 'dst_sha1': match.group(4),
573 'status': match.group(5),
574 'status_score': match.group(6),
575 'src': match.group(7),
576 'dst': match.group(10)
578 return None
580 def isModeExec(mode):
581 # Returns True if the given git mode represents an executable file,
582 # otherwise False.
583 return mode[-3:] == "755"
585 class P4Exception(Exception):
586 """ Base class for exceptions from the p4 client """
587 def __init__(self, exit_code):
588 self.p4ExitCode = exit_code
590 class P4ServerException(P4Exception):
591 """ Base class for exceptions where we get some kind of marshalled up result from the server """
592 def __init__(self, exit_code, p4_result):
593 super(P4ServerException, self).__init__(exit_code)
594 self.p4_result = p4_result
595 self.code = p4_result[0]['code']
596 self.data = p4_result[0]['data']
598 class P4RequestSizeException(P4ServerException):
599 """ One of the maxresults or maxscanrows errors """
600 def __init__(self, exit_code, p4_result, limit):
601 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
602 self.limit = limit
604 def isModeExecChanged(src_mode, dst_mode):
605 return isModeExec(src_mode) != isModeExec(dst_mode)
607 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
608 errors_as_exceptions=False):
610 if isinstance(cmd,basestring):
611 cmd = "-G " + cmd
612 expand = True
613 else:
614 cmd = ["-G"] + cmd
615 expand = False
617 cmd = p4_build_cmd(cmd)
618 if verbose:
619 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
621 # Use a temporary file to avoid deadlocks without
622 # subprocess.communicate(), which would put another copy
623 # of stdout into memory.
624 stdin_file = None
625 if stdin is not None:
626 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
627 if isinstance(stdin,basestring):
628 stdin_file.write(stdin)
629 else:
630 for i in stdin:
631 stdin_file.write(i + '\n')
632 stdin_file.flush()
633 stdin_file.seek(0)
635 p4 = subprocess.Popen(cmd,
636 shell=expand,
637 stdin=stdin_file,
638 stdout=subprocess.PIPE)
640 result = []
641 try:
642 while True:
643 entry = marshal.load(p4.stdout)
644 if skip_info:
645 if 'code' in entry and entry['code'] == 'info':
646 continue
647 if cb is not None:
648 cb(entry)
649 else:
650 result.append(entry)
651 except EOFError:
652 pass
653 exitCode = p4.wait()
654 if exitCode != 0:
655 if errors_as_exceptions:
656 if len(result) > 0:
657 data = result[0].get('data')
658 if data:
659 m = re.search('Too many rows scanned \(over (\d+)\)', data)
660 if not m:
661 m = re.search('Request too large \(over (\d+)\)', data)
663 if m:
664 limit = int(m.group(1))
665 raise P4RequestSizeException(exitCode, result, limit)
667 raise P4ServerException(exitCode, result)
668 else:
669 raise P4Exception(exitCode)
670 else:
671 entry = {}
672 entry["p4ExitCode"] = exitCode
673 result.append(entry)
675 return result
677 def p4Cmd(cmd):
678 list = p4CmdList(cmd)
679 result = {}
680 for entry in list:
681 result.update(entry)
682 return result;
684 def p4Where(depotPath):
685 if not depotPath.endswith("/"):
686 depotPath += "/"
687 depotPathLong = depotPath + "..."
688 outputList = p4CmdList(["where", depotPathLong])
689 output = None
690 for entry in outputList:
691 if "depotFile" in entry:
692 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
693 # The base path always ends with "/...".
694 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
695 output = entry
696 break
697 elif "data" in entry:
698 data = entry.get("data")
699 space = data.find(" ")
700 if data[:space] == depotPath:
701 output = entry
702 break
703 if output == None:
704 return ""
705 if output["code"] == "error":
706 return ""
707 clientPath = ""
708 if "path" in output:
709 clientPath = output.get("path")
710 elif "data" in output:
711 data = output.get("data")
712 lastSpace = data.rfind(" ")
713 clientPath = data[lastSpace + 1:]
715 if clientPath.endswith("..."):
716 clientPath = clientPath[:-3]
717 return clientPath
719 def currentGitBranch():
720 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
722 def isValidGitDir(path):
723 return git_dir(path) != None
725 def parseRevision(ref):
726 return read_pipe("git rev-parse %s" % ref).strip()
728 def branchExists(ref):
729 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
730 ignore_error=True)
731 return len(rev) > 0
733 def extractLogMessageFromGitCommit(commit):
734 logMessage = ""
736 ## fixme: title is first line of commit, not 1st paragraph.
737 foundTitle = False
738 for log in read_pipe_lines("git cat-file commit %s" % commit):
739 if not foundTitle:
740 if len(log) == 1:
741 foundTitle = True
742 continue
744 logMessage += log
745 return logMessage
747 def extractSettingsGitLog(log):
748 values = {}
749 for line in log.split("\n"):
750 line = line.strip()
751 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
752 if not m:
753 continue
755 assignments = m.group(1).split (':')
756 for a in assignments:
757 vals = a.split ('=')
758 key = vals[0].strip()
759 val = ('='.join (vals[1:])).strip()
760 if val.endswith ('\"') and val.startswith('"'):
761 val = val[1:-1]
763 values[key] = val
765 paths = values.get("depot-paths")
766 if not paths:
767 paths = values.get("depot-path")
768 if paths:
769 values['depot-paths'] = paths.split(',')
770 return values
772 def gitBranchExists(branch):
773 proc = subprocess.Popen(["git", "rev-parse", branch],
774 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
775 return proc.wait() == 0;
777 def gitUpdateRef(ref, newvalue):
778 subprocess.check_call(["git", "update-ref", ref, newvalue])
780 def gitDeleteRef(ref):
781 subprocess.check_call(["git", "update-ref", "-d", ref])
783 _gitConfig = {}
785 def gitConfig(key, typeSpecifier=None):
786 if key not in _gitConfig:
787 cmd = [ "git", "config" ]
788 if typeSpecifier:
789 cmd += [ typeSpecifier ]
790 cmd += [ key ]
791 s = read_pipe(cmd, ignore_error=True)
792 _gitConfig[key] = s.strip()
793 return _gitConfig[key]
795 def gitConfigBool(key):
796 """Return a bool, using git config --bool. It is True only if the
797 variable is set to true, and False if set to false or not present
798 in the config."""
800 if key not in _gitConfig:
801 _gitConfig[key] = gitConfig(key, '--bool') == "true"
802 return _gitConfig[key]
804 def gitConfigInt(key):
805 if key not in _gitConfig:
806 cmd = [ "git", "config", "--int", key ]
807 s = read_pipe(cmd, ignore_error=True)
808 v = s.strip()
809 try:
810 _gitConfig[key] = int(gitConfig(key, '--int'))
811 except ValueError:
812 _gitConfig[key] = None
813 return _gitConfig[key]
815 def gitConfigList(key):
816 if key not in _gitConfig:
817 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
818 _gitConfig[key] = s.strip().splitlines()
819 if _gitConfig[key] == ['']:
820 _gitConfig[key] = []
821 return _gitConfig[key]
823 def p4BranchesInGit(branchesAreInRemotes=True):
824 """Find all the branches whose names start with "p4/", looking
825 in remotes or heads as specified by the argument. Return
826 a dictionary of { branch: revision } for each one found.
827 The branch names are the short names, without any
828 "p4/" prefix."""
830 branches = {}
832 cmdline = "git rev-parse --symbolic "
833 if branchesAreInRemotes:
834 cmdline += "--remotes"
835 else:
836 cmdline += "--branches"
838 for line in read_pipe_lines(cmdline):
839 line = line.strip()
841 # only import to p4/
842 if not line.startswith('p4/'):
843 continue
844 # special symbolic ref to p4/master
845 if line == "p4/HEAD":
846 continue
848 # strip off p4/ prefix
849 branch = line[len("p4/"):]
851 branches[branch] = parseRevision(line)
853 return branches
855 def branch_exists(branch):
856 """Make sure that the given ref name really exists."""
858 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
859 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
860 out, _ = p.communicate()
861 if p.returncode:
862 return False
863 # expect exactly one line of output: the branch name
864 return out.rstrip() == branch
866 def findUpstreamBranchPoint(head = "HEAD"):
867 branches = p4BranchesInGit()
868 # map from depot-path to branch name
869 branchByDepotPath = {}
870 for branch in branches.keys():
871 tip = branches[branch]
872 log = extractLogMessageFromGitCommit(tip)
873 settings = extractSettingsGitLog(log)
874 if "depot-paths" in settings:
875 paths = ",".join(settings["depot-paths"])
876 branchByDepotPath[paths] = "remotes/p4/" + branch
878 settings = None
879 parent = 0
880 while parent < 65535:
881 commit = head + "~%s" % parent
882 log = extractLogMessageFromGitCommit(commit)
883 settings = extractSettingsGitLog(log)
884 if "depot-paths" in settings:
885 paths = ",".join(settings["depot-paths"])
886 if paths in branchByDepotPath:
887 return [branchByDepotPath[paths], settings]
889 parent = parent + 1
891 return ["", settings]
893 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
894 if not silent:
895 print("Creating/updating branch(es) in %s based on origin branch(es)"
896 % localRefPrefix)
898 originPrefix = "origin/p4/"
900 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
901 line = line.strip()
902 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
903 continue
905 headName = line[len(originPrefix):]
906 remoteHead = localRefPrefix + headName
907 originHead = line
909 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
910 if ('depot-paths' not in original
911 or 'change' not in original):
912 continue
914 update = False
915 if not gitBranchExists(remoteHead):
916 if verbose:
917 print("creating %s" % remoteHead)
918 update = True
919 else:
920 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
921 if 'change' in settings:
922 if settings['depot-paths'] == original['depot-paths']:
923 originP4Change = int(original['change'])
924 p4Change = int(settings['change'])
925 if originP4Change > p4Change:
926 print("%s (%s) is newer than %s (%s). "
927 "Updating p4 branch from origin."
928 % (originHead, originP4Change,
929 remoteHead, p4Change))
930 update = True
931 else:
932 print("Ignoring: %s was imported from %s while "
933 "%s was imported from %s"
934 % (originHead, ','.join(original['depot-paths']),
935 remoteHead, ','.join(settings['depot-paths'])))
937 if update:
938 system("git update-ref %s %s" % (remoteHead, originHead))
940 def originP4BranchesExist():
941 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
944 def p4ParseNumericChangeRange(parts):
945 changeStart = int(parts[0][1:])
946 if parts[1] == '#head':
947 changeEnd = p4_last_change()
948 else:
949 changeEnd = int(parts[1])
951 return (changeStart, changeEnd)
953 def chooseBlockSize(blockSize):
954 if blockSize:
955 return blockSize
956 else:
957 return defaultBlockSize
959 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
960 assert depotPaths
962 # Parse the change range into start and end. Try to find integer
963 # revision ranges as these can be broken up into blocks to avoid
964 # hitting server-side limits (maxrows, maxscanresults). But if
965 # that doesn't work, fall back to using the raw revision specifier
966 # strings, without using block mode.
968 if changeRange is None or changeRange == '':
969 changeStart = 1
970 changeEnd = p4_last_change()
971 block_size = chooseBlockSize(requestedBlockSize)
972 else:
973 parts = changeRange.split(',')
974 assert len(parts) == 2
975 try:
976 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
977 block_size = chooseBlockSize(requestedBlockSize)
978 except ValueError:
979 changeStart = parts[0][1:]
980 changeEnd = parts[1]
981 if requestedBlockSize:
982 die("cannot use --changes-block-size with non-numeric revisions")
983 block_size = None
985 changes = set()
987 # Retrieve changes a block at a time, to prevent running
988 # into a MaxResults/MaxScanRows error from the server. If
989 # we _do_ hit one of those errors, turn down the block size
991 while True:
992 cmd = ['changes']
994 if block_size:
995 end = min(changeEnd, changeStart + block_size)
996 revisionRange = "%d,%d" % (changeStart, end)
997 else:
998 revisionRange = "%s,%s" % (changeStart, changeEnd)
1000 for p in depotPaths:
1001 cmd += ["%s...@%s" % (p, revisionRange)]
1003 # fetch the changes
1004 try:
1005 result = p4CmdList(cmd, errors_as_exceptions=True)
1006 except P4RequestSizeException as e:
1007 if not block_size:
1008 block_size = e.limit
1009 elif block_size > e.limit:
1010 block_size = e.limit
1011 else:
1012 block_size = max(2, block_size // 2)
1014 if verbose: print("block size error, retrying with block size {0}".format(block_size))
1015 continue
1016 except P4Exception as e:
1017 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1019 # Insert changes in chronological order
1020 for entry in reversed(result):
1021 if 'change' not in entry:
1022 continue
1023 changes.add(int(entry['change']))
1025 if not block_size:
1026 break
1028 if end >= changeEnd:
1029 break
1031 changeStart = end + 1
1033 changes = sorted(changes)
1034 return changes
1036 def p4PathStartsWith(path, prefix):
1037 # This method tries to remedy a potential mixed-case issue:
1039 # If UserA adds //depot/DirA/file1
1040 # and UserB adds //depot/dira/file2
1042 # we may or may not have a problem. If you have core.ignorecase=true,
1043 # we treat DirA and dira as the same directory
1044 if gitConfigBool("core.ignorecase"):
1045 return path.lower().startswith(prefix.lower())
1046 return path.startswith(prefix)
1048 def getClientSpec():
1049 """Look at the p4 client spec, create a View() object that contains
1050 all the mappings, and return it."""
1052 specList = p4CmdList("client -o")
1053 if len(specList) != 1:
1054 die('Output from "client -o" is %d lines, expecting 1' %
1055 len(specList))
1057 # dictionary of all client parameters
1058 entry = specList[0]
1060 # the //client/ name
1061 client_name = entry["Client"]
1063 # just the keys that start with "View"
1064 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1066 # hold this new View
1067 view = View(client_name)
1069 # append the lines, in order, to the view
1070 for view_num in range(len(view_keys)):
1071 k = "View%d" % view_num
1072 if k not in view_keys:
1073 die("Expected view key %s missing" % k)
1074 view.append(entry[k])
1076 return view
1078 def getClientRoot():
1079 """Grab the client directory."""
1081 output = p4CmdList("client -o")
1082 if len(output) != 1:
1083 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1085 entry = output[0]
1086 if "Root" not in entry:
1087 die('Client has no "Root"')
1089 return entry["Root"]
1092 # P4 wildcards are not allowed in filenames. P4 complains
1093 # if you simply add them, but you can force it with "-f", in
1094 # which case it translates them into %xx encoding internally.
1096 def wildcard_decode(path):
1097 # Search for and fix just these four characters. Do % last so
1098 # that fixing it does not inadvertently create new %-escapes.
1099 # Cannot have * in a filename in windows; untested as to
1100 # what p4 would do in such a case.
1101 if not platform.system() == "Windows":
1102 path = path.replace("%2A", "*")
1103 path = path.replace("%23", "#") \
1104 .replace("%40", "@") \
1105 .replace("%25", "%")
1106 return path
1108 def wildcard_encode(path):
1109 # do % first to avoid double-encoding the %s introduced here
1110 path = path.replace("%", "%25") \
1111 .replace("*", "%2A") \
1112 .replace("#", "%23") \
1113 .replace("@", "%40")
1114 return path
1116 def wildcard_present(path):
1117 m = re.search("[*#@%]", path)
1118 return m is not None
1120 class LargeFileSystem(object):
1121 """Base class for large file system support."""
1123 def __init__(self, writeToGitStream):
1124 self.largeFiles = set()
1125 self.writeToGitStream = writeToGitStream
1127 def generatePointer(self, cloneDestination, contentFile):
1128 """Return the content of a pointer file that is stored in Git instead of
1129 the actual content."""
1130 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1132 def pushFile(self, localLargeFile):
1133 """Push the actual content which is not stored in the Git repository to
1134 a server."""
1135 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1137 def hasLargeFileExtension(self, relPath):
1138 return reduce(
1139 lambda a, b: a or b,
1140 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1141 False
1144 def generateTempFile(self, contents):
1145 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1146 for d in contents:
1147 contentFile.write(d)
1148 contentFile.close()
1149 return contentFile.name
1151 def exceedsLargeFileThreshold(self, relPath, contents):
1152 if gitConfigInt('git-p4.largeFileThreshold'):
1153 contentsSize = sum(len(d) for d in contents)
1154 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1155 return True
1156 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1157 contentsSize = sum(len(d) for d in contents)
1158 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159 return False
1160 contentTempFile = self.generateTempFile(contents)
1161 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1162 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1163 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1164 zf.close()
1165 compressedContentsSize = zf.infolist()[0].compress_size
1166 os.remove(contentTempFile)
1167 os.remove(compressedContentFile.name)
1168 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1169 return True
1170 return False
1172 def addLargeFile(self, relPath):
1173 self.largeFiles.add(relPath)
1175 def removeLargeFile(self, relPath):
1176 self.largeFiles.remove(relPath)
1178 def isLargeFile(self, relPath):
1179 return relPath in self.largeFiles
1181 def processContent(self, git_mode, relPath, contents):
1182 """Processes the content of git fast import. This method decides if a
1183 file is stored in the large file system and handles all necessary
1184 steps."""
1185 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1186 contentTempFile = self.generateTempFile(contents)
1187 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1188 if pointer_git_mode:
1189 git_mode = pointer_git_mode
1190 if localLargeFile:
1191 # Move temp file to final location in large file system
1192 largeFileDir = os.path.dirname(localLargeFile)
1193 if not os.path.isdir(largeFileDir):
1194 os.makedirs(largeFileDir)
1195 shutil.move(contentTempFile, localLargeFile)
1196 self.addLargeFile(relPath)
1197 if gitConfigBool('git-p4.largeFilePush'):
1198 self.pushFile(localLargeFile)
1199 if verbose:
1200 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1201 return (git_mode, contents)
1203 class MockLFS(LargeFileSystem):
1204 """Mock large file system for testing."""
1206 def generatePointer(self, contentFile):
1207 """The pointer content is the original content prefixed with "pointer-".
1208 The local filename of the large file storage is derived from the file content.
1210 with open(contentFile, 'r') as f:
1211 content = next(f)
1212 gitMode = '100644'
1213 pointerContents = 'pointer-' + content
1214 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1215 return (gitMode, pointerContents, localLargeFile)
1217 def pushFile(self, localLargeFile):
1218 """The remote filename of the large file storage is the same as the local
1219 one but in a different directory.
1221 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1222 if not os.path.exists(remotePath):
1223 os.makedirs(remotePath)
1224 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1226 class GitLFS(LargeFileSystem):
1227 """Git LFS as backend for the git-p4 large file system.
1228 See https://git-lfs.github.com/ for details."""
1230 def __init__(self, *args):
1231 LargeFileSystem.__init__(self, *args)
1232 self.baseGitAttributes = []
1234 def generatePointer(self, contentFile):
1235 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1236 mode and content which is stored in the Git repository instead of
1237 the actual content. Return also the new location of the actual
1238 content.
1240 if os.path.getsize(contentFile) == 0:
1241 return (None, '', None)
1243 pointerProcess = subprocess.Popen(
1244 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1245 stdout=subprocess.PIPE
1247 pointerFile = pointerProcess.stdout.read()
1248 if pointerProcess.wait():
1249 os.remove(contentFile)
1250 die('git-lfs pointer command failed. Did you install the extension?')
1252 # Git LFS removed the preamble in the output of the 'pointer' command
1253 # starting from version 1.2.0. Check for the preamble here to support
1254 # earlier versions.
1255 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1256 if pointerFile.startswith('Git LFS pointer for'):
1257 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1259 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1260 localLargeFile = os.path.join(
1261 os.getcwd(),
1262 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1263 oid,
1265 # LFS Spec states that pointer files should not have the executable bit set.
1266 gitMode = '100644'
1267 return (gitMode, pointerFile, localLargeFile)
1269 def pushFile(self, localLargeFile):
1270 uploadProcess = subprocess.Popen(
1271 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1273 if uploadProcess.wait():
1274 die('git-lfs push command failed. Did you define a remote?')
1276 def generateGitAttributes(self):
1277 return (
1278 self.baseGitAttributes +
1280 '\n',
1281 '#\n',
1282 '# Git LFS (see https://git-lfs.github.com/)\n',
1283 '#\n',
1285 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1286 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1288 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1289 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1293 def addLargeFile(self, relPath):
1294 LargeFileSystem.addLargeFile(self, relPath)
1295 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1297 def removeLargeFile(self, relPath):
1298 LargeFileSystem.removeLargeFile(self, relPath)
1299 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1301 def processContent(self, git_mode, relPath, contents):
1302 if relPath == '.gitattributes':
1303 self.baseGitAttributes = contents
1304 return (git_mode, self.generateGitAttributes())
1305 else:
1306 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1308 class Command:
1309 def __init__(self):
1310 self.usage = "usage: %prog [options]"
1311 self.needsGit = True
1312 self.verbose = False
1314 # This is required for the "append" cloneExclude action
1315 def ensure_value(self, attr, value):
1316 if not hasattr(self, attr) or getattr(self, attr) is None:
1317 setattr(self, attr, value)
1318 return getattr(self, attr)
1320 class P4UserMap:
1321 def __init__(self):
1322 self.userMapFromPerforceServer = False
1323 self.myP4UserId = None
1325 def p4UserId(self):
1326 if self.myP4UserId:
1327 return self.myP4UserId
1329 results = p4CmdList("user -o")
1330 for r in results:
1331 if 'User' in r:
1332 self.myP4UserId = r['User']
1333 return r['User']
1334 die("Could not find your p4 user id")
1336 def p4UserIsMe(self, p4User):
1337 # return True if the given p4 user is actually me
1338 me = self.p4UserId()
1339 if not p4User or p4User != me:
1340 return False
1341 else:
1342 return True
1344 def getUserCacheFilename(self):
1345 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1346 return home + "/.gitp4-usercache.txt"
1348 def getUserMapFromPerforceServer(self):
1349 if self.userMapFromPerforceServer:
1350 return
1351 self.users = {}
1352 self.emails = {}
1354 for output in p4CmdList("users"):
1355 if "User" not in output:
1356 continue
1357 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1358 self.emails[output["Email"]] = output["User"]
1360 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1361 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1362 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1363 if mapUser and len(mapUser[0]) == 3:
1364 user = mapUser[0][0]
1365 fullname = mapUser[0][1]
1366 email = mapUser[0][2]
1367 self.users[user] = fullname + " <" + email + ">"
1368 self.emails[email] = user
1370 s = ''
1371 for (key, val) in self.users.items():
1372 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1374 open(self.getUserCacheFilename(), "wb").write(s)
1375 self.userMapFromPerforceServer = True
1377 def loadUserMapFromCache(self):
1378 self.users = {}
1379 self.userMapFromPerforceServer = False
1380 try:
1381 cache = open(self.getUserCacheFilename(), "rb")
1382 lines = cache.readlines()
1383 cache.close()
1384 for line in lines:
1385 entry = line.strip().split("\t")
1386 self.users[entry[0]] = entry[1]
1387 except IOError:
1388 self.getUserMapFromPerforceServer()
1390 class P4Debug(Command):
1391 def __init__(self):
1392 Command.__init__(self)
1393 self.options = []
1394 self.description = "A tool to debug the output of p4 -G."
1395 self.needsGit = False
1397 def run(self, args):
1398 j = 0
1399 for output in p4CmdList(args):
1400 print('Element: %d' % j)
1401 j += 1
1402 print(output)
1403 return True
1405 class P4RollBack(Command):
1406 def __init__(self):
1407 Command.__init__(self)
1408 self.options = [
1409 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1411 self.description = "A tool to debug the multi-branch import. Don't use :)"
1412 self.rollbackLocalBranches = False
1414 def run(self, args):
1415 if len(args) != 1:
1416 return False
1417 maxChange = int(args[0])
1419 if "p4ExitCode" in p4Cmd("changes -m 1"):
1420 die("Problems executing p4");
1422 if self.rollbackLocalBranches:
1423 refPrefix = "refs/heads/"
1424 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1425 else:
1426 refPrefix = "refs/remotes/"
1427 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1429 for line in lines:
1430 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1431 line = line.strip()
1432 ref = refPrefix + line
1433 log = extractLogMessageFromGitCommit(ref)
1434 settings = extractSettingsGitLog(log)
1436 depotPaths = settings['depot-paths']
1437 change = settings['change']
1439 changed = False
1441 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1442 for p in depotPaths]))) == 0:
1443 print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1444 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1445 continue
1447 while change and int(change) > maxChange:
1448 changed = True
1449 if self.verbose:
1450 print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1451 system("git update-ref %s \"%s^\"" % (ref, ref))
1452 log = extractLogMessageFromGitCommit(ref)
1453 settings = extractSettingsGitLog(log)
1456 depotPaths = settings['depot-paths']
1457 change = settings['change']
1459 if changed:
1460 print("%s rewound to %s" % (ref, change))
1462 return True
1464 class P4Submit(Command, P4UserMap):
1466 conflict_behavior_choices = ("ask", "skip", "quit")
1468 def __init__(self):
1469 Command.__init__(self)
1470 P4UserMap.__init__(self)
1471 self.options = [
1472 optparse.make_option("--origin", dest="origin"),
1473 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1474 # preserve the user, requires relevant p4 permissions
1475 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1476 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1477 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1478 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1479 optparse.make_option("--conflict", dest="conflict_behavior",
1480 choices=self.conflict_behavior_choices),
1481 optparse.make_option("--branch", dest="branch"),
1482 optparse.make_option("--shelve", dest="shelve", action="store_true",
1483 help="Shelve instead of submit. Shelved files are reverted, "
1484 "restoring the workspace to the state before the shelve"),
1485 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1486 metavar="CHANGELIST",
1487 help="update an existing shelved changelist, implies --shelve, "
1488 "repeat in-order for multiple shelved changelists"),
1489 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1490 help="submit only the specified commit(s), one commit or xxx..xxx"),
1491 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1492 help="Disable rebase after submit is completed. Can be useful if you "
1493 "work from a local git branch that is not master"),
1494 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1495 help="Skip Perforce sync of p4/master after submit or shelve"),
1497 self.description = """Submit changes from git to the perforce depot.\n
1498 The `p4-pre-submit` hook is executed if it exists and is executable.
1499 The hook takes no parameters and nothing from standard input. Exiting with
1500 non-zero status from this script prevents `git-p4 submit` from launching.
1502 One usage scenario is to run unit tests in the hook."""
1504 self.usage += " [name of git branch to submit into perforce depot]"
1505 self.origin = ""
1506 self.detectRenames = False
1507 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1508 self.dry_run = False
1509 self.shelve = False
1510 self.update_shelve = list()
1511 self.commit = ""
1512 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1513 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1514 self.prepare_p4_only = False
1515 self.conflict_behavior = None
1516 self.isWindows = (platform.system() == "Windows")
1517 self.exportLabels = False
1518 self.p4HasMoveCommand = p4_has_move_command()
1519 self.branch = None
1521 if gitConfig('git-p4.largeFileSystem'):
1522 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1524 def check(self):
1525 if len(p4CmdList("opened ...")) > 0:
1526 die("You have files opened with perforce! Close them before starting the sync.")
1528 def separate_jobs_from_description(self, message):
1529 """Extract and return a possible Jobs field in the commit
1530 message. It goes into a separate section in the p4 change
1531 specification.
1533 A jobs line starts with "Jobs:" and looks like a new field
1534 in a form. Values are white-space separated on the same
1535 line or on following lines that start with a tab.
1537 This does not parse and extract the full git commit message
1538 like a p4 form. It just sees the Jobs: line as a marker
1539 to pass everything from then on directly into the p4 form,
1540 but outside the description section.
1542 Return a tuple (stripped log message, jobs string)."""
1544 m = re.search(r'^Jobs:', message, re.MULTILINE)
1545 if m is None:
1546 return (message, None)
1548 jobtext = message[m.start():]
1549 stripped_message = message[:m.start()].rstrip()
1550 return (stripped_message, jobtext)
1552 def prepareLogMessage(self, template, message, jobs):
1553 """Edits the template returned from "p4 change -o" to insert
1554 the message in the Description field, and the jobs text in
1555 the Jobs field."""
1556 result = ""
1558 inDescriptionSection = False
1560 for line in template.split("\n"):
1561 if line.startswith("#"):
1562 result += line + "\n"
1563 continue
1565 if inDescriptionSection:
1566 if line.startswith("Files:") or line.startswith("Jobs:"):
1567 inDescriptionSection = False
1568 # insert Jobs section
1569 if jobs:
1570 result += jobs + "\n"
1571 else:
1572 continue
1573 else:
1574 if line.startswith("Description:"):
1575 inDescriptionSection = True
1576 line += "\n"
1577 for messageLine in message.split("\n"):
1578 line += "\t" + messageLine + "\n"
1580 result += line + "\n"
1582 return result
1584 def patchRCSKeywords(self, file, pattern):
1585 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1586 (handle, outFileName) = tempfile.mkstemp(dir='.')
1587 try:
1588 outFile = os.fdopen(handle, "w+")
1589 inFile = open(file, "r")
1590 regexp = re.compile(pattern, re.VERBOSE)
1591 for line in inFile.readlines():
1592 line = regexp.sub(r'$\1$', line)
1593 outFile.write(line)
1594 inFile.close()
1595 outFile.close()
1596 # Forcibly overwrite the original file
1597 os.unlink(file)
1598 shutil.move(outFileName, file)
1599 except:
1600 # cleanup our temporary file
1601 os.unlink(outFileName)
1602 print("Failed to strip RCS keywords in %s" % file)
1603 raise
1605 print("Patched up RCS keywords in %s" % file)
1607 def p4UserForCommit(self,id):
1608 # Return the tuple (perforce user,git email) for a given git commit id
1609 self.getUserMapFromPerforceServer()
1610 gitEmail = read_pipe(["git", "log", "--max-count=1",
1611 "--format=%ae", id])
1612 gitEmail = gitEmail.strip()
1613 if gitEmail not in self.emails:
1614 return (None,gitEmail)
1615 else:
1616 return (self.emails[gitEmail],gitEmail)
1618 def checkValidP4Users(self,commits):
1619 # check if any git authors cannot be mapped to p4 users
1620 for id in commits:
1621 (user,email) = self.p4UserForCommit(id)
1622 if not user:
1623 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1624 if gitConfigBool("git-p4.allowMissingP4Users"):
1625 print("%s" % msg)
1626 else:
1627 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1629 def lastP4Changelist(self):
1630 # Get back the last changelist number submitted in this client spec. This
1631 # then gets used to patch up the username in the change. If the same
1632 # client spec is being used by multiple processes then this might go
1633 # wrong.
1634 results = p4CmdList("client -o") # find the current client
1635 client = None
1636 for r in results:
1637 if 'Client' in r:
1638 client = r['Client']
1639 break
1640 if not client:
1641 die("could not get client spec")
1642 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1643 for r in results:
1644 if 'change' in r:
1645 return r['change']
1646 die("Could not get changelist number for last submit - cannot patch up user details")
1648 def modifyChangelistUser(self, changelist, newUser):
1649 # fixup the user field of a changelist after it has been submitted.
1650 changes = p4CmdList("change -o %s" % changelist)
1651 if len(changes) != 1:
1652 die("Bad output from p4 change modifying %s to user %s" %
1653 (changelist, newUser))
1655 c = changes[0]
1656 if c['User'] == newUser: return # nothing to do
1657 c['User'] = newUser
1658 input = marshal.dumps(c)
1660 result = p4CmdList("change -f -i", stdin=input)
1661 for r in result:
1662 if 'code' in r:
1663 if r['code'] == 'error':
1664 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1665 if 'data' in r:
1666 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1667 return
1668 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1670 def canChangeChangelists(self):
1671 # check to see if we have p4 admin or super-user permissions, either of
1672 # which are required to modify changelists.
1673 results = p4CmdList(["protects", self.depotPath])
1674 for r in results:
1675 if 'perm' in r:
1676 if r['perm'] == 'admin':
1677 return 1
1678 if r['perm'] == 'super':
1679 return 1
1680 return 0
1682 def prepareSubmitTemplate(self, changelist=None):
1683 """Run "p4 change -o" to grab a change specification template.
1684 This does not use "p4 -G", as it is nice to keep the submission
1685 template in original order, since a human might edit it.
1687 Remove lines in the Files section that show changes to files
1688 outside the depot path we're committing into."""
1690 [upstream, settings] = findUpstreamBranchPoint()
1692 template = """\
1693 # A Perforce Change Specification.
1695 # Change: The change number. 'new' on a new changelist.
1696 # Date: The date this specification was last modified.
1697 # Client: The client on which the changelist was created. Read-only.
1698 # User: The user who created the changelist.
1699 # Status: Either 'pending' or 'submitted'. Read-only.
1700 # Type: Either 'public' or 'restricted'. Default is 'public'.
1701 # Description: Comments about the changelist. Required.
1702 # Jobs: What opened jobs are to be closed by this changelist.
1703 # You may delete jobs from this list. (New changelists only.)
1704 # Files: What opened files from the default changelist are to be added
1705 # to this changelist. You may delete files from this list.
1706 # (New changelists only.)
1708 files_list = []
1709 inFilesSection = False
1710 change_entry = None
1711 args = ['change', '-o']
1712 if changelist:
1713 args.append(str(changelist))
1714 for entry in p4CmdList(args):
1715 if 'code' not in entry:
1716 continue
1717 if entry['code'] == 'stat':
1718 change_entry = entry
1719 break
1720 if not change_entry:
1721 die('Failed to decode output of p4 change -o')
1722 for key, value in change_entry.iteritems():
1723 if key.startswith('File'):
1724 if 'depot-paths' in settings:
1725 if not [p for p in settings['depot-paths']
1726 if p4PathStartsWith(value, p)]:
1727 continue
1728 else:
1729 if not p4PathStartsWith(value, self.depotPath):
1730 continue
1731 files_list.append(value)
1732 continue
1733 # Output in the order expected by prepareLogMessage
1734 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1735 if key not in change_entry:
1736 continue
1737 template += '\n'
1738 template += key + ':'
1739 if key == 'Description':
1740 template += '\n'
1741 for field_line in change_entry[key].splitlines():
1742 template += '\t'+field_line+'\n'
1743 if len(files_list) > 0:
1744 template += '\n'
1745 template += 'Files:\n'
1746 for path in files_list:
1747 template += '\t'+path+'\n'
1748 return template
1750 def edit_template(self, template_file):
1751 """Invoke the editor to let the user change the submission
1752 message. Return true if okay to continue with the submit."""
1754 # if configured to skip the editing part, just submit
1755 if gitConfigBool("git-p4.skipSubmitEdit"):
1756 return True
1758 # look at the modification time, to check later if the user saved
1759 # the file
1760 mtime = os.stat(template_file).st_mtime
1762 # invoke the editor
1763 if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1764 editor = os.environ.get("P4EDITOR")
1765 else:
1766 editor = read_pipe("git var GIT_EDITOR").strip()
1767 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1769 # If the file was not saved, prompt to see if this patch should
1770 # be skipped. But skip this verification step if configured so.
1771 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1772 return True
1774 # modification time updated means user saved the file
1775 if os.stat(template_file).st_mtime > mtime:
1776 return True
1778 while True:
1779 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1780 if response == 'y':
1781 return True
1782 if response == 'n':
1783 return False
1785 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1786 # diff
1787 if "P4DIFF" in os.environ:
1788 del(os.environ["P4DIFF"])
1789 diff = ""
1790 for editedFile in editedFiles:
1791 diff += p4_read_pipe(['diff', '-du',
1792 wildcard_encode(editedFile)])
1794 # new file diff
1795 newdiff = ""
1796 for newFile in filesToAdd:
1797 newdiff += "==== new file ====\n"
1798 newdiff += "--- /dev/null\n"
1799 newdiff += "+++ %s\n" % newFile
1801 is_link = os.path.islink(newFile)
1802 expect_link = newFile in symlinks
1804 if is_link and expect_link:
1805 newdiff += "+%s\n" % os.readlink(newFile)
1806 else:
1807 f = open(newFile, "r")
1808 for line in f.readlines():
1809 newdiff += "+" + line
1810 f.close()
1812 return (diff + newdiff).replace('\r\n', '\n')
1814 def applyCommit(self, id):
1815 """Apply one commit, return True if it succeeded."""
1817 print("Applying", read_pipe(["git", "show", "-s",
1818 "--format=format:%h %s", id]))
1820 (p4User, gitEmail) = self.p4UserForCommit(id)
1822 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1823 filesToAdd = set()
1824 filesToChangeType = set()
1825 filesToDelete = set()
1826 editedFiles = set()
1827 pureRenameCopy = set()
1828 symlinks = set()
1829 filesToChangeExecBit = {}
1830 all_files = list()
1832 for line in diff:
1833 diff = parseDiffTreeEntry(line)
1834 modifier = diff['status']
1835 path = diff['src']
1836 all_files.append(path)
1838 if modifier == "M":
1839 p4_edit(path)
1840 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1841 filesToChangeExecBit[path] = diff['dst_mode']
1842 editedFiles.add(path)
1843 elif modifier == "A":
1844 filesToAdd.add(path)
1845 filesToChangeExecBit[path] = diff['dst_mode']
1846 if path in filesToDelete:
1847 filesToDelete.remove(path)
1849 dst_mode = int(diff['dst_mode'], 8)
1850 if dst_mode == 0o120000:
1851 symlinks.add(path)
1853 elif modifier == "D":
1854 filesToDelete.add(path)
1855 if path in filesToAdd:
1856 filesToAdd.remove(path)
1857 elif modifier == "C":
1858 src, dest = diff['src'], diff['dst']
1859 p4_integrate(src, dest)
1860 pureRenameCopy.add(dest)
1861 if diff['src_sha1'] != diff['dst_sha1']:
1862 p4_edit(dest)
1863 pureRenameCopy.discard(dest)
1864 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1865 p4_edit(dest)
1866 pureRenameCopy.discard(dest)
1867 filesToChangeExecBit[dest] = diff['dst_mode']
1868 if self.isWindows:
1869 # turn off read-only attribute
1870 os.chmod(dest, stat.S_IWRITE)
1871 os.unlink(dest)
1872 editedFiles.add(dest)
1873 elif modifier == "R":
1874 src, dest = diff['src'], diff['dst']
1875 if self.p4HasMoveCommand:
1876 p4_edit(src) # src must be open before move
1877 p4_move(src, dest) # opens for (move/delete, move/add)
1878 else:
1879 p4_integrate(src, dest)
1880 if diff['src_sha1'] != diff['dst_sha1']:
1881 p4_edit(dest)
1882 else:
1883 pureRenameCopy.add(dest)
1884 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1885 if not self.p4HasMoveCommand:
1886 p4_edit(dest) # with move: already open, writable
1887 filesToChangeExecBit[dest] = diff['dst_mode']
1888 if not self.p4HasMoveCommand:
1889 if self.isWindows:
1890 os.chmod(dest, stat.S_IWRITE)
1891 os.unlink(dest)
1892 filesToDelete.add(src)
1893 editedFiles.add(dest)
1894 elif modifier == "T":
1895 filesToChangeType.add(path)
1896 else:
1897 die("unknown modifier %s for %s" % (modifier, path))
1899 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1900 patchcmd = diffcmd + " | git apply "
1901 tryPatchCmd = patchcmd + "--check -"
1902 applyPatchCmd = patchcmd + "--check --apply -"
1903 patch_succeeded = True
1905 if os.system(tryPatchCmd) != 0:
1906 fixed_rcs_keywords = False
1907 patch_succeeded = False
1908 print("Unfortunately applying the change failed!")
1910 # Patch failed, maybe it's just RCS keyword woes. Look through
1911 # the patch to see if that's possible.
1912 if gitConfigBool("git-p4.attemptRCSCleanup"):
1913 file = None
1914 pattern = None
1915 kwfiles = {}
1916 for file in editedFiles | filesToDelete:
1917 # did this file's delta contain RCS keywords?
1918 pattern = p4_keywords_regexp_for_file(file)
1920 if pattern:
1921 # this file is a possibility...look for RCS keywords.
1922 regexp = re.compile(pattern, re.VERBOSE)
1923 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1924 if regexp.search(line):
1925 if verbose:
1926 print("got keyword match on %s in %s in %s" % (pattern, line, file))
1927 kwfiles[file] = pattern
1928 break
1930 for file in kwfiles:
1931 if verbose:
1932 print("zapping %s with %s" % (line,pattern))
1933 # File is being deleted, so not open in p4. Must
1934 # disable the read-only bit on windows.
1935 if self.isWindows and file not in editedFiles:
1936 os.chmod(file, stat.S_IWRITE)
1937 self.patchRCSKeywords(file, kwfiles[file])
1938 fixed_rcs_keywords = True
1940 if fixed_rcs_keywords:
1941 print("Retrying the patch with RCS keywords cleaned up")
1942 if os.system(tryPatchCmd) == 0:
1943 patch_succeeded = True
1945 if not patch_succeeded:
1946 for f in editedFiles:
1947 p4_revert(f)
1948 return False
1951 # Apply the patch for real, and do add/delete/+x handling.
1953 system(applyPatchCmd)
1955 for f in filesToChangeType:
1956 p4_edit(f, "-t", "auto")
1957 for f in filesToAdd:
1958 p4_add(f)
1959 for f in filesToDelete:
1960 p4_revert(f)
1961 p4_delete(f)
1963 # Set/clear executable bits
1964 for f in filesToChangeExecBit.keys():
1965 mode = filesToChangeExecBit[f]
1966 setP4ExecBit(f, mode)
1968 update_shelve = 0
1969 if len(self.update_shelve) > 0:
1970 update_shelve = self.update_shelve.pop(0)
1971 p4_reopen_in_change(update_shelve, all_files)
1974 # Build p4 change description, starting with the contents
1975 # of the git commit message.
1977 logMessage = extractLogMessageFromGitCommit(id)
1978 logMessage = logMessage.strip()
1979 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1981 template = self.prepareSubmitTemplate(update_shelve)
1982 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1984 if self.preserveUser:
1985 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1987 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1988 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1989 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1990 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1992 separatorLine = "######## everything below this line is just the diff #######\n"
1993 if not self.prepare_p4_only:
1994 submitTemplate += separatorLine
1995 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1997 (handle, fileName) = tempfile.mkstemp()
1998 tmpFile = os.fdopen(handle, "w+b")
1999 if self.isWindows:
2000 submitTemplate = submitTemplate.replace("\n", "\r\n")
2001 tmpFile.write(submitTemplate)
2002 tmpFile.close()
2004 if self.prepare_p4_only:
2006 # Leave the p4 tree prepared, and the submit template around
2007 # and let the user decide what to do next
2009 print()
2010 print("P4 workspace prepared for submission.")
2011 print("To submit or revert, go to client workspace")
2012 print(" " + self.clientPath)
2013 print()
2014 print("To submit, use \"p4 submit\" to write a new description,")
2015 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2016 " \"git p4\"." % fileName)
2017 print("You can delete the file \"%s\" when finished." % fileName)
2019 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2020 print("To preserve change ownership by user %s, you must\n" \
2021 "do \"p4 change -f <change>\" after submitting and\n" \
2022 "edit the User field.")
2023 if pureRenameCopy:
2024 print("After submitting, renamed files must be re-synced.")
2025 print("Invoke \"p4 sync -f\" on each of these files:")
2026 for f in pureRenameCopy:
2027 print(" " + f)
2029 print()
2030 print("To revert the changes, use \"p4 revert ...\", and delete")
2031 print("the submit template file \"%s\"" % fileName)
2032 if filesToAdd:
2033 print("Since the commit adds new files, they must be deleted:")
2034 for f in filesToAdd:
2035 print(" " + f)
2036 print()
2037 return True
2040 # Let the user edit the change description, then submit it.
2042 submitted = False
2044 try:
2045 if self.edit_template(fileName):
2046 # read the edited message and submit
2047 tmpFile = open(fileName, "rb")
2048 message = tmpFile.read()
2049 tmpFile.close()
2050 if self.isWindows:
2051 message = message.replace("\r\n", "\n")
2052 submitTemplate = message[:message.index(separatorLine)]
2054 if update_shelve:
2055 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2056 elif self.shelve:
2057 p4_write_pipe(['shelve', '-i'], submitTemplate)
2058 else:
2059 p4_write_pipe(['submit', '-i'], submitTemplate)
2060 # The rename/copy happened by applying a patch that created a
2061 # new file. This leaves it writable, which confuses p4.
2062 for f in pureRenameCopy:
2063 p4_sync(f, "-f")
2065 if self.preserveUser:
2066 if p4User:
2067 # Get last changelist number. Cannot easily get it from
2068 # the submit command output as the output is
2069 # unmarshalled.
2070 changelist = self.lastP4Changelist()
2071 self.modifyChangelistUser(changelist, p4User)
2073 submitted = True
2075 finally:
2076 # skip this patch
2077 if not submitted or self.shelve:
2078 if self.shelve:
2079 print ("Reverting shelved files.")
2080 else:
2081 print ("Submission cancelled, undoing p4 changes.")
2082 for f in editedFiles | filesToDelete:
2083 p4_revert(f)
2084 for f in filesToAdd:
2085 p4_revert(f)
2086 os.remove(f)
2088 os.remove(fileName)
2089 return submitted
2091 # Export git tags as p4 labels. Create a p4 label and then tag
2092 # with that.
2093 def exportGitTags(self, gitTags):
2094 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2095 if len(validLabelRegexp) == 0:
2096 validLabelRegexp = defaultLabelRegexp
2097 m = re.compile(validLabelRegexp)
2099 for name in gitTags:
2101 if not m.match(name):
2102 if verbose:
2103 print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2104 continue
2106 # Get the p4 commit this corresponds to
2107 logMessage = extractLogMessageFromGitCommit(name)
2108 values = extractSettingsGitLog(logMessage)
2110 if 'change' not in values:
2111 # a tag pointing to something not sent to p4; ignore
2112 if verbose:
2113 print("git tag %s does not give a p4 commit" % name)
2114 continue
2115 else:
2116 changelist = values['change']
2118 # Get the tag details.
2119 inHeader = True
2120 isAnnotated = False
2121 body = []
2122 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2123 l = l.strip()
2124 if inHeader:
2125 if re.match(r'tag\s+', l):
2126 isAnnotated = True
2127 elif re.match(r'\s*$', l):
2128 inHeader = False
2129 continue
2130 else:
2131 body.append(l)
2133 if not isAnnotated:
2134 body = ["lightweight tag imported by git p4\n"]
2136 # Create the label - use the same view as the client spec we are using
2137 clientSpec = getClientSpec()
2139 labelTemplate = "Label: %s\n" % name
2140 labelTemplate += "Description:\n"
2141 for b in body:
2142 labelTemplate += "\t" + b + "\n"
2143 labelTemplate += "View:\n"
2144 for depot_side in clientSpec.mappings:
2145 labelTemplate += "\t%s\n" % depot_side
2147 if self.dry_run:
2148 print("Would create p4 label %s for tag" % name)
2149 elif self.prepare_p4_only:
2150 print("Not creating p4 label %s for tag due to option" \
2151 " --prepare-p4-only" % name)
2152 else:
2153 p4_write_pipe(["label", "-i"], labelTemplate)
2155 # Use the label
2156 p4_system(["tag", "-l", name] +
2157 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2159 if verbose:
2160 print("created p4 label for tag %s" % name)
2162 def run(self, args):
2163 if len(args) == 0:
2164 self.master = currentGitBranch()
2165 elif len(args) == 1:
2166 self.master = args[0]
2167 if not branchExists(self.master):
2168 die("Branch %s does not exist" % self.master)
2169 else:
2170 return False
2172 for i in self.update_shelve:
2173 if i <= 0:
2174 sys.exit("invalid changelist %d" % i)
2176 if self.master:
2177 allowSubmit = gitConfig("git-p4.allowSubmit")
2178 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2179 die("%s is not in git-p4.allowSubmit" % self.master)
2181 [upstream, settings] = findUpstreamBranchPoint()
2182 self.depotPath = settings['depot-paths'][0]
2183 if len(self.origin) == 0:
2184 self.origin = upstream
2186 if len(self.update_shelve) > 0:
2187 self.shelve = True
2189 if self.preserveUser:
2190 if not self.canChangeChangelists():
2191 die("Cannot preserve user names without p4 super-user or admin permissions")
2193 # if not set from the command line, try the config file
2194 if self.conflict_behavior is None:
2195 val = gitConfig("git-p4.conflict")
2196 if val:
2197 if val not in self.conflict_behavior_choices:
2198 die("Invalid value '%s' for config git-p4.conflict" % val)
2199 else:
2200 val = "ask"
2201 self.conflict_behavior = val
2203 if self.verbose:
2204 print("Origin branch is " + self.origin)
2206 if len(self.depotPath) == 0:
2207 print("Internal error: cannot locate perforce depot path from existing branches")
2208 sys.exit(128)
2210 self.useClientSpec = False
2211 if gitConfigBool("git-p4.useclientspec"):
2212 self.useClientSpec = True
2213 if self.useClientSpec:
2214 self.clientSpecDirs = getClientSpec()
2216 # Check for the existence of P4 branches
2217 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2219 if self.useClientSpec and not branchesDetected:
2220 # all files are relative to the client spec
2221 self.clientPath = getClientRoot()
2222 else:
2223 self.clientPath = p4Where(self.depotPath)
2225 if self.clientPath == "":
2226 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2228 print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2229 self.oldWorkingDirectory = os.getcwd()
2231 # ensure the clientPath exists
2232 new_client_dir = False
2233 if not os.path.exists(self.clientPath):
2234 new_client_dir = True
2235 os.makedirs(self.clientPath)
2237 chdir(self.clientPath, is_client_path=True)
2238 if self.dry_run:
2239 print("Would synchronize p4 checkout in %s" % self.clientPath)
2240 else:
2241 print("Synchronizing p4 checkout...")
2242 if new_client_dir:
2243 # old one was destroyed, and maybe nobody told p4
2244 p4_sync("...", "-f")
2245 else:
2246 p4_sync("...")
2247 self.check()
2249 commits = []
2250 if self.master:
2251 committish = self.master
2252 else:
2253 committish = 'HEAD'
2255 if self.commit != "":
2256 if self.commit.find("..") != -1:
2257 limits_ish = self.commit.split("..")
2258 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2259 commits.append(line.strip())
2260 commits.reverse()
2261 else:
2262 commits.append(self.commit)
2263 else:
2264 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2265 commits.append(line.strip())
2266 commits.reverse()
2268 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2269 self.checkAuthorship = False
2270 else:
2271 self.checkAuthorship = True
2273 if self.preserveUser:
2274 self.checkValidP4Users(commits)
2277 # Build up a set of options to be passed to diff when
2278 # submitting each commit to p4.
2280 if self.detectRenames:
2281 # command-line -M arg
2282 self.diffOpts = "-M"
2283 else:
2284 # If not explicitly set check the config variable
2285 detectRenames = gitConfig("git-p4.detectRenames")
2287 if detectRenames.lower() == "false" or detectRenames == "":
2288 self.diffOpts = ""
2289 elif detectRenames.lower() == "true":
2290 self.diffOpts = "-M"
2291 else:
2292 self.diffOpts = "-M%s" % detectRenames
2294 # no command-line arg for -C or --find-copies-harder, just
2295 # config variables
2296 detectCopies = gitConfig("git-p4.detectCopies")
2297 if detectCopies.lower() == "false" or detectCopies == "":
2298 pass
2299 elif detectCopies.lower() == "true":
2300 self.diffOpts += " -C"
2301 else:
2302 self.diffOpts += " -C%s" % detectCopies
2304 if gitConfigBool("git-p4.detectCopiesHarder"):
2305 self.diffOpts += " --find-copies-harder"
2307 num_shelves = len(self.update_shelve)
2308 if num_shelves > 0 and num_shelves != len(commits):
2309 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2310 (len(commits), num_shelves))
2312 hooks_path = gitConfig("core.hooksPath")
2313 if len(hooks_path) <= 0:
2314 hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2316 hook_file = os.path.join(hooks_path, "p4-pre-submit")
2317 if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2318 sys.exit(1)
2321 # Apply the commits, one at a time. On failure, ask if should
2322 # continue to try the rest of the patches, or quit.
2324 if self.dry_run:
2325 print("Would apply")
2326 applied = []
2327 last = len(commits) - 1
2328 for i, commit in enumerate(commits):
2329 if self.dry_run:
2330 print(" ", read_pipe(["git", "show", "-s",
2331 "--format=format:%h %s", commit]))
2332 ok = True
2333 else:
2334 ok = self.applyCommit(commit)
2335 if ok:
2336 applied.append(commit)
2337 else:
2338 if self.prepare_p4_only and i < last:
2339 print("Processing only the first commit due to option" \
2340 " --prepare-p4-only")
2341 break
2342 if i < last:
2343 quit = False
2344 while True:
2345 # prompt for what to do, or use the option/variable
2346 if self.conflict_behavior == "ask":
2347 print("What do you want to do?")
2348 response = raw_input("[s]kip this commit but apply"
2349 " the rest, or [q]uit? ")
2350 if not response:
2351 continue
2352 elif self.conflict_behavior == "skip":
2353 response = "s"
2354 elif self.conflict_behavior == "quit":
2355 response = "q"
2356 else:
2357 die("Unknown conflict_behavior '%s'" %
2358 self.conflict_behavior)
2360 if response[0] == "s":
2361 print("Skipping this commit, but applying the rest")
2362 break
2363 if response[0] == "q":
2364 print("Quitting")
2365 quit = True
2366 break
2367 if quit:
2368 break
2370 chdir(self.oldWorkingDirectory)
2371 shelved_applied = "shelved" if self.shelve else "applied"
2372 if self.dry_run:
2373 pass
2374 elif self.prepare_p4_only:
2375 pass
2376 elif len(commits) == len(applied):
2377 print("All commits {0}!".format(shelved_applied))
2379 sync = P4Sync()
2380 if self.branch:
2381 sync.branch = self.branch
2382 if self.disable_p4sync:
2383 sync.sync_origin_only()
2384 else:
2385 sync.run([])
2387 if not self.disable_rebase:
2388 rebase = P4Rebase()
2389 rebase.rebase()
2391 else:
2392 if len(applied) == 0:
2393 print("No commits {0}.".format(shelved_applied))
2394 else:
2395 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2396 for c in commits:
2397 if c in applied:
2398 star = "*"
2399 else:
2400 star = " "
2401 print(star, read_pipe(["git", "show", "-s",
2402 "--format=format:%h %s", c]))
2403 print("You will have to do 'git p4 sync' and rebase.")
2405 if gitConfigBool("git-p4.exportLabels"):
2406 self.exportLabels = True
2408 if self.exportLabels:
2409 p4Labels = getP4Labels(self.depotPath)
2410 gitTags = getGitTags()
2412 missingGitTags = gitTags - p4Labels
2413 self.exportGitTags(missingGitTags)
2415 # exit with error unless everything applied perfectly
2416 if len(commits) != len(applied):
2417 sys.exit(1)
2419 return True
2421 class View(object):
2422 """Represent a p4 view ("p4 help views"), and map files in a
2423 repo according to the view."""
2425 def __init__(self, client_name):
2426 self.mappings = []
2427 self.client_prefix = "//%s/" % client_name
2428 # cache results of "p4 where" to lookup client file locations
2429 self.client_spec_path_cache = {}
2431 def append(self, view_line):
2432 """Parse a view line, splitting it into depot and client
2433 sides. Append to self.mappings, preserving order. This
2434 is only needed for tag creation."""
2436 # Split the view line into exactly two words. P4 enforces
2437 # structure on these lines that simplifies this quite a bit.
2439 # Either or both words may be double-quoted.
2440 # Single quotes do not matter.
2441 # Double-quote marks cannot occur inside the words.
2442 # A + or - prefix is also inside the quotes.
2443 # There are no quotes unless they contain a space.
2444 # The line is already white-space stripped.
2445 # The two words are separated by a single space.
2447 if view_line[0] == '"':
2448 # First word is double quoted. Find its end.
2449 close_quote_index = view_line.find('"', 1)
2450 if close_quote_index <= 0:
2451 die("No first-word closing quote found: %s" % view_line)
2452 depot_side = view_line[1:close_quote_index]
2453 # skip closing quote and space
2454 rhs_index = close_quote_index + 1 + 1
2455 else:
2456 space_index = view_line.find(" ")
2457 if space_index <= 0:
2458 die("No word-splitting space found: %s" % view_line)
2459 depot_side = view_line[0:space_index]
2460 rhs_index = space_index + 1
2462 # prefix + means overlay on previous mapping
2463 if depot_side.startswith("+"):
2464 depot_side = depot_side[1:]
2466 # prefix - means exclude this path, leave out of mappings
2467 exclude = False
2468 if depot_side.startswith("-"):
2469 exclude = True
2470 depot_side = depot_side[1:]
2472 if not exclude:
2473 self.mappings.append(depot_side)
2475 def convert_client_path(self, clientFile):
2476 # chop off //client/ part to make it relative
2477 if not clientFile.startswith(self.client_prefix):
2478 die("No prefix '%s' on clientFile '%s'" %
2479 (self.client_prefix, clientFile))
2480 return clientFile[len(self.client_prefix):]
2482 def update_client_spec_path_cache(self, files):
2483 """ Caching file paths by "p4 where" batch query """
2485 # List depot file paths exclude that already cached
2486 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2488 if len(fileArgs) == 0:
2489 return # All files in cache
2491 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2492 for res in where_result:
2493 if "code" in res and res["code"] == "error":
2494 # assume error is "... file(s) not in client view"
2495 continue
2496 if "clientFile" not in res:
2497 die("No clientFile in 'p4 where' output")
2498 if "unmap" in res:
2499 # it will list all of them, but only one not unmap-ped
2500 continue
2501 if gitConfigBool("core.ignorecase"):
2502 res['depotFile'] = res['depotFile'].lower()
2503 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2505 # not found files or unmap files set to ""
2506 for depotFile in fileArgs:
2507 if gitConfigBool("core.ignorecase"):
2508 depotFile = depotFile.lower()
2509 if depotFile not in self.client_spec_path_cache:
2510 self.client_spec_path_cache[depotFile] = ""
2512 def map_in_client(self, depot_path):
2513 """Return the relative location in the client where this
2514 depot file should live. Returns "" if the file should
2515 not be mapped in the client."""
2517 if gitConfigBool("core.ignorecase"):
2518 depot_path = depot_path.lower()
2520 if depot_path in self.client_spec_path_cache:
2521 return self.client_spec_path_cache[depot_path]
2523 die( "Error: %s is not found in client spec path" % depot_path )
2524 return ""
2526 class P4Sync(Command, P4UserMap):
2527 delete_actions = ( "delete", "move/delete", "purge" )
2529 def __init__(self):
2530 Command.__init__(self)
2531 P4UserMap.__init__(self)
2532 self.options = [
2533 optparse.make_option("--branch", dest="branch"),
2534 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2535 optparse.make_option("--changesfile", dest="changesFile"),
2536 optparse.make_option("--silent", dest="silent", action="store_true"),
2537 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2538 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2539 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2540 help="Import into refs/heads/ , not refs/remotes"),
2541 optparse.make_option("--max-changes", dest="maxChanges",
2542 help="Maximum number of changes to import"),
2543 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2544 help="Internal block size to use when iteratively calling p4 changes"),
2545 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2546 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2547 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2548 help="Only sync files that are included in the Perforce Client Spec"),
2549 optparse.make_option("-/", dest="cloneExclude",
2550 action="append", type="string",
2551 help="exclude depot path"),
2553 self.description = """Imports from Perforce into a git repository.\n
2554 example:
2555 //depot/my/project/ -- to import the current head
2556 //depot/my/project/@all -- to import everything
2557 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2559 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2561 self.usage += " //depot/path[@revRange]"
2562 self.silent = False
2563 self.createdBranches = set()
2564 self.committedChanges = set()
2565 self.branch = ""
2566 self.detectBranches = False
2567 self.detectLabels = False
2568 self.importLabels = False
2569 self.changesFile = ""
2570 self.syncWithOrigin = True
2571 self.importIntoRemotes = True
2572 self.maxChanges = ""
2573 self.changes_block_size = None
2574 self.keepRepoPath = False
2575 self.depotPaths = None
2576 self.p4BranchesInGit = []
2577 self.cloneExclude = []
2578 self.useClientSpec = False
2579 self.useClientSpec_from_options = False
2580 self.clientSpecDirs = None
2581 self.tempBranches = []
2582 self.tempBranchLocation = "refs/git-p4-tmp"
2583 self.largeFileSystem = None
2584 self.suppress_meta_comment = False
2586 if gitConfig('git-p4.largeFileSystem'):
2587 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2588 self.largeFileSystem = largeFileSystemConstructor(
2589 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2592 if gitConfig("git-p4.syncFromOrigin") == "false":
2593 self.syncWithOrigin = False
2595 self.depotPaths = []
2596 self.changeRange = ""
2597 self.previousDepotPaths = []
2598 self.hasOrigin = False
2600 # map from branch depot path to parent branch
2601 self.knownBranches = {}
2602 self.initialParents = {}
2604 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2605 self.labels = {}
2607 # Force a checkpoint in fast-import and wait for it to finish
2608 def checkpoint(self):
2609 self.gitStream.write("checkpoint\n\n")
2610 self.gitStream.write("progress checkpoint\n\n")
2611 out = self.gitOutput.readline()
2612 if self.verbose:
2613 print("checkpoint finished: " + out)
2615 def cmp_shelved(self, path, filerev, revision):
2616 """ Determine if a path at revision #filerev is the same as the file
2617 at revision @revision for a shelved changelist. If they don't match,
2618 unshelving won't be safe (we will get other changes mixed in).
2620 This is comparing the revision that the shelved changelist is *based* on, not
2621 the shelved changelist itself.
2623 ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)])
2624 if verbose:
2625 print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret))
2626 return ret["status"] == "identical"
2628 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):
2629 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2630 for path in self.cloneExclude]
2631 files = []
2632 fnum = 0
2633 while "depotFile%s" % fnum in commit:
2634 path = commit["depotFile%s" % fnum]
2636 if [p for p in self.cloneExclude
2637 if p4PathStartsWith(path, p)]:
2638 found = False
2639 else:
2640 found = [p for p in self.depotPaths
2641 if p4PathStartsWith(path, p)]
2642 if not found:
2643 fnum = fnum + 1
2644 continue
2646 file = {}
2647 file["path"] = path
2648 file["rev"] = commit["rev%s" % fnum]
2649 file["action"] = commit["action%s" % fnum]
2650 file["type"] = commit["type%s" % fnum]
2651 if shelved:
2652 file["shelved_cl"] = int(shelved_cl)
2654 # For shelved changelists, check that the revision of each file that the
2655 # shelve was based on matches the revision that we are using for the
2656 # starting point for git-fast-import (self.initialParent). Otherwise
2657 # the resulting diff will contain deltas from multiple commits.
2659 if file["action"] != "add" and \
2660 not self.cmp_shelved(path, file["rev"], origin_revision):
2661 sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2662 commit["change"], self.initialParent, path))
2664 files.append(file)
2665 fnum = fnum + 1
2666 return files
2668 def extractJobsFromCommit(self, commit):
2669 jobs = []
2670 jnum = 0
2671 while "job%s" % jnum in commit:
2672 job = commit["job%s" % jnum]
2673 jobs.append(job)
2674 jnum = jnum + 1
2675 return jobs
2677 def stripRepoPath(self, path, prefixes):
2678 """When streaming files, this is called to map a p4 depot path
2679 to where it should go in git. The prefixes are either
2680 self.depotPaths, or self.branchPrefixes in the case of
2681 branch detection."""
2683 if self.useClientSpec:
2684 # branch detection moves files up a level (the branch name)
2685 # from what client spec interpretation gives
2686 path = self.clientSpecDirs.map_in_client(path)
2687 if self.detectBranches:
2688 for b in self.knownBranches:
2689 if path.startswith(b + "/"):
2690 path = path[len(b)+1:]
2692 elif self.keepRepoPath:
2693 # Preserve everything in relative path name except leading
2694 # //depot/; just look at first prefix as they all should
2695 # be in the same depot.
2696 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2697 if p4PathStartsWith(path, depot):
2698 path = path[len(depot):]
2700 else:
2701 for p in prefixes:
2702 if p4PathStartsWith(path, p):
2703 path = path[len(p):]
2704 break
2706 path = wildcard_decode(path)
2707 return path
2709 def splitFilesIntoBranches(self, commit):
2710 """Look at each depotFile in the commit to figure out to what
2711 branch it belongs."""
2713 if self.clientSpecDirs:
2714 files = self.extractFilesFromCommit(commit)
2715 self.clientSpecDirs.update_client_spec_path_cache(files)
2717 branches = {}
2718 fnum = 0
2719 while "depotFile%s" % fnum in commit:
2720 path = commit["depotFile%s" % fnum]
2721 found = [p for p in self.depotPaths
2722 if p4PathStartsWith(path, p)]
2723 if not found:
2724 fnum = fnum + 1
2725 continue
2727 file = {}
2728 file["path"] = path
2729 file["rev"] = commit["rev%s" % fnum]
2730 file["action"] = commit["action%s" % fnum]
2731 file["type"] = commit["type%s" % fnum]
2732 fnum = fnum + 1
2734 # start with the full relative path where this file would
2735 # go in a p4 client
2736 if self.useClientSpec:
2737 relPath = self.clientSpecDirs.map_in_client(path)
2738 else:
2739 relPath = self.stripRepoPath(path, self.depotPaths)
2741 for branch in self.knownBranches.keys():
2742 # add a trailing slash so that a commit into qt/4.2foo
2743 # doesn't end up in qt/4.2, e.g.
2744 if relPath.startswith(branch + "/"):
2745 if branch not in branches:
2746 branches[branch] = []
2747 branches[branch].append(file)
2748 break
2750 return branches
2752 def writeToGitStream(self, gitMode, relPath, contents):
2753 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2754 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2755 for d in contents:
2756 self.gitStream.write(d)
2757 self.gitStream.write('\n')
2759 def encodeWithUTF8(self, path):
2760 try:
2761 path.decode('ascii')
2762 except:
2763 encoding = 'utf8'
2764 if gitConfig('git-p4.pathEncoding'):
2765 encoding = gitConfig('git-p4.pathEncoding')
2766 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2767 if self.verbose:
2768 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2769 return path
2771 # output one file from the P4 stream
2772 # - helper for streamP4Files
2774 def streamOneP4File(self, file, contents):
2775 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2776 relPath = self.encodeWithUTF8(relPath)
2777 if verbose:
2778 size = int(self.stream_file['fileSize'])
2779 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2780 sys.stdout.flush()
2782 (type_base, type_mods) = split_p4_type(file["type"])
2784 git_mode = "100644"
2785 if "x" in type_mods:
2786 git_mode = "100755"
2787 if type_base == "symlink":
2788 git_mode = "120000"
2789 # p4 print on a symlink sometimes contains "target\n";
2790 # if it does, remove the newline
2791 data = ''.join(contents)
2792 if not data:
2793 # Some version of p4 allowed creating a symlink that pointed
2794 # to nothing. This causes p4 errors when checking out such
2795 # a change, and errors here too. Work around it by ignoring
2796 # the bad symlink; hopefully a future change fixes it.
2797 print("\nIgnoring empty symlink in %s" % file['depotFile'])
2798 return
2799 elif data[-1] == '\n':
2800 contents = [data[:-1]]
2801 else:
2802 contents = [data]
2804 if type_base == "utf16":
2805 # p4 delivers different text in the python output to -G
2806 # than it does when using "print -o", or normal p4 client
2807 # operations. utf16 is converted to ascii or utf8, perhaps.
2808 # But ascii text saved as -t utf16 is completely mangled.
2809 # Invoke print -o to get the real contents.
2811 # On windows, the newlines will always be mangled by print, so put
2812 # them back too. This is not needed to the cygwin windows version,
2813 # just the native "NT" type.
2815 try:
2816 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2817 except Exception as e:
2818 if 'Translation of file content failed' in str(e):
2819 type_base = 'binary'
2820 else:
2821 raise e
2822 else:
2823 if p4_version_string().find('/NT') >= 0:
2824 text = text.replace('\r\n', '\n')
2825 contents = [ text ]
2827 if type_base == "apple":
2828 # Apple filetype files will be streamed as a concatenation of
2829 # its appledouble header and the contents. This is useless
2830 # on both macs and non-macs. If using "print -q -o xx", it
2831 # will create "xx" with the data, and "%xx" with the header.
2832 # This is also not very useful.
2834 # Ideally, someday, this script can learn how to generate
2835 # appledouble files directly and import those to git, but
2836 # non-mac machines can never find a use for apple filetype.
2837 print("\nIgnoring apple filetype file %s" % file['depotFile'])
2838 return
2840 # Note that we do not try to de-mangle keywords on utf16 files,
2841 # even though in theory somebody may want that.
2842 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2843 if pattern:
2844 regexp = re.compile(pattern, re.VERBOSE)
2845 text = ''.join(contents)
2846 text = regexp.sub(r'$\1$', text)
2847 contents = [ text ]
2849 if self.largeFileSystem:
2850 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2852 self.writeToGitStream(git_mode, relPath, contents)
2854 def streamOneP4Deletion(self, file):
2855 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2856 relPath = self.encodeWithUTF8(relPath)
2857 if verbose:
2858 sys.stdout.write("delete %s\n" % relPath)
2859 sys.stdout.flush()
2860 self.gitStream.write("D %s\n" % relPath)
2862 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2863 self.largeFileSystem.removeLargeFile(relPath)
2865 # handle another chunk of streaming data
2866 def streamP4FilesCb(self, marshalled):
2868 # catch p4 errors and complain
2869 err = None
2870 if "code" in marshalled:
2871 if marshalled["code"] == "error":
2872 if "data" in marshalled:
2873 err = marshalled["data"].rstrip()
2875 if not err and 'fileSize' in self.stream_file:
2876 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2877 if required_bytes > 0:
2878 err = 'Not enough space left on %s! Free at least %i MB.' % (
2879 os.getcwd(), required_bytes/1024/1024
2882 if err:
2883 f = None
2884 if self.stream_have_file_info:
2885 if "depotFile" in self.stream_file:
2886 f = self.stream_file["depotFile"]
2887 # force a failure in fast-import, else an empty
2888 # commit will be made
2889 self.gitStream.write("\n")
2890 self.gitStream.write("die-now\n")
2891 self.gitStream.close()
2892 # ignore errors, but make sure it exits first
2893 self.importProcess.wait()
2894 if f:
2895 die("Error from p4 print for %s: %s" % (f, err))
2896 else:
2897 die("Error from p4 print: %s" % err)
2899 if 'depotFile' in marshalled and self.stream_have_file_info:
2900 # start of a new file - output the old one first
2901 self.streamOneP4File(self.stream_file, self.stream_contents)
2902 self.stream_file = {}
2903 self.stream_contents = []
2904 self.stream_have_file_info = False
2906 # pick up the new file information... for the
2907 # 'data' field we need to append to our array
2908 for k in marshalled.keys():
2909 if k == 'data':
2910 if 'streamContentSize' not in self.stream_file:
2911 self.stream_file['streamContentSize'] = 0
2912 self.stream_file['streamContentSize'] += len(marshalled['data'])
2913 self.stream_contents.append(marshalled['data'])
2914 else:
2915 self.stream_file[k] = marshalled[k]
2917 if (verbose and
2918 'streamContentSize' in self.stream_file and
2919 'fileSize' in self.stream_file and
2920 'depotFile' in self.stream_file):
2921 size = int(self.stream_file["fileSize"])
2922 if size > 0:
2923 progress = 100*self.stream_file['streamContentSize']/size
2924 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2925 sys.stdout.flush()
2927 self.stream_have_file_info = True
2929 # Stream directly from "p4 files" into "git fast-import"
2930 def streamP4Files(self, files):
2931 filesForCommit = []
2932 filesToRead = []
2933 filesToDelete = []
2935 for f in files:
2936 filesForCommit.append(f)
2937 if f['action'] in self.delete_actions:
2938 filesToDelete.append(f)
2939 else:
2940 filesToRead.append(f)
2942 # deleted files...
2943 for f in filesToDelete:
2944 self.streamOneP4Deletion(f)
2946 if len(filesToRead) > 0:
2947 self.stream_file = {}
2948 self.stream_contents = []
2949 self.stream_have_file_info = False
2951 # curry self argument
2952 def streamP4FilesCbSelf(entry):
2953 self.streamP4FilesCb(entry)
2955 fileArgs = []
2956 for f in filesToRead:
2957 if 'shelved_cl' in f:
2958 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2959 # the contents
2960 fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2961 else:
2962 fileArg = '%s#%s' % (f['path'], f['rev'])
2964 fileArgs.append(fileArg)
2966 p4CmdList(["-x", "-", "print"],
2967 stdin=fileArgs,
2968 cb=streamP4FilesCbSelf)
2970 # do the last chunk
2971 if 'depotFile' in self.stream_file:
2972 self.streamOneP4File(self.stream_file, self.stream_contents)
2974 def make_email(self, userid):
2975 if userid in self.users:
2976 return self.users[userid]
2977 else:
2978 return "%s <a@b>" % userid
2980 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2981 """ Stream a p4 tag.
2982 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2985 if verbose:
2986 print("writing tag %s for commit %s" % (labelName, commit))
2987 gitStream.write("tag %s\n" % labelName)
2988 gitStream.write("from %s\n" % commit)
2990 if 'Owner' in labelDetails:
2991 owner = labelDetails["Owner"]
2992 else:
2993 owner = None
2995 # Try to use the owner of the p4 label, or failing that,
2996 # the current p4 user id.
2997 if owner:
2998 email = self.make_email(owner)
2999 else:
3000 email = self.make_email(self.p4UserId())
3001 tagger = "%s %s %s" % (email, epoch, self.tz)
3003 gitStream.write("tagger %s\n" % tagger)
3005 print("labelDetails=",labelDetails)
3006 if 'Description' in labelDetails:
3007 description = labelDetails['Description']
3008 else:
3009 description = 'Label from git p4'
3011 gitStream.write("data %d\n" % len(description))
3012 gitStream.write(description)
3013 gitStream.write("\n")
3015 def inClientSpec(self, path):
3016 if not self.clientSpecDirs:
3017 return True
3018 inClientSpec = self.clientSpecDirs.map_in_client(path)
3019 if not inClientSpec and self.verbose:
3020 print('Ignoring file outside of client spec: {0}'.format(path))
3021 return inClientSpec
3023 def hasBranchPrefix(self, path):
3024 if not self.branchPrefixes:
3025 return True
3026 hasPrefix = [p for p in self.branchPrefixes
3027 if p4PathStartsWith(path, p)]
3028 if not hasPrefix and self.verbose:
3029 print('Ignoring file outside of prefix: {0}'.format(path))
3030 return hasPrefix
3032 def commit(self, details, files, branch, parent = ""):
3033 epoch = details["time"]
3034 author = details["user"]
3035 jobs = self.extractJobsFromCommit(details)
3037 if self.verbose:
3038 print('commit into {0}'.format(branch))
3040 if self.clientSpecDirs:
3041 self.clientSpecDirs.update_client_spec_path_cache(files)
3043 files = [f for f in files
3044 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3046 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
3047 print('Ignoring revision {0} as it would produce an empty commit.'
3048 .format(details['change']))
3049 return
3051 self.gitStream.write("commit %s\n" % branch)
3052 self.gitStream.write("mark :%s\n" % details["change"])
3053 self.committedChanges.add(int(details["change"]))
3054 committer = ""
3055 if author not in self.users:
3056 self.getUserMapFromPerforceServer()
3057 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3059 self.gitStream.write("committer %s\n" % committer)
3061 self.gitStream.write("data <<EOT\n")
3062 self.gitStream.write(details["desc"])
3063 if len(jobs) > 0:
3064 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3066 if not self.suppress_meta_comment:
3067 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3068 (','.join(self.branchPrefixes), details["change"]))
3069 if len(details['options']) > 0:
3070 self.gitStream.write(": options = %s" % details['options'])
3071 self.gitStream.write("]\n")
3073 self.gitStream.write("EOT\n\n")
3075 if len(parent) > 0:
3076 if self.verbose:
3077 print("parent %s" % parent)
3078 self.gitStream.write("from %s\n" % parent)
3080 self.streamP4Files(files)
3081 self.gitStream.write("\n")
3083 change = int(details["change"])
3085 if change in self.labels:
3086 label = self.labels[change]
3087 labelDetails = label[0]
3088 labelRevisions = label[1]
3089 if self.verbose:
3090 print("Change %s is labelled %s" % (change, labelDetails))
3092 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3093 for p in self.branchPrefixes])
3095 if len(files) == len(labelRevisions):
3097 cleanedFiles = {}
3098 for info in files:
3099 if info["action"] in self.delete_actions:
3100 continue
3101 cleanedFiles[info["depotFile"]] = info["rev"]
3103 if cleanedFiles == labelRevisions:
3104 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3106 else:
3107 if not self.silent:
3108 print("Tag %s does not match with change %s: files do not match."
3109 % (labelDetails["label"], change))
3111 else:
3112 if not self.silent:
3113 print("Tag %s does not match with change %s: file count is different."
3114 % (labelDetails["label"], change))
3116 # Build a dictionary of changelists and labels, for "detect-labels" option.
3117 def getLabels(self):
3118 self.labels = {}
3120 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3121 if len(l) > 0 and not self.silent:
3122 print("Finding files belonging to labels in %s" % self.depotPaths)
3124 for output in l:
3125 label = output["label"]
3126 revisions = {}
3127 newestChange = 0
3128 if self.verbose:
3129 print("Querying files for label %s" % label)
3130 for file in p4CmdList(["files"] +
3131 ["%s...@%s" % (p, label)
3132 for p in self.depotPaths]):
3133 revisions[file["depotFile"]] = file["rev"]
3134 change = int(file["change"])
3135 if change > newestChange:
3136 newestChange = change
3138 self.labels[newestChange] = [output, revisions]
3140 if self.verbose:
3141 print("Label changes: %s" % self.labels.keys())
3143 # Import p4 labels as git tags. A direct mapping does not
3144 # exist, so assume that if all the files are at the same revision
3145 # then we can use that, or it's something more complicated we should
3146 # just ignore.
3147 def importP4Labels(self, stream, p4Labels):
3148 if verbose:
3149 print("import p4 labels: " + ' '.join(p4Labels))
3151 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3152 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3153 if len(validLabelRegexp) == 0:
3154 validLabelRegexp = defaultLabelRegexp
3155 m = re.compile(validLabelRegexp)
3157 for name in p4Labels:
3158 commitFound = False
3160 if not m.match(name):
3161 if verbose:
3162 print("label %s does not match regexp %s" % (name,validLabelRegexp))
3163 continue
3165 if name in ignoredP4Labels:
3166 continue
3168 labelDetails = p4CmdList(['label', "-o", name])[0]
3170 # get the most recent changelist for each file in this label
3171 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3172 for p in self.depotPaths])
3174 if 'change' in change:
3175 # find the corresponding git commit; take the oldest commit
3176 changelist = int(change['change'])
3177 if changelist in self.committedChanges:
3178 gitCommit = ":%d" % changelist # use a fast-import mark
3179 commitFound = True
3180 else:
3181 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3182 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3183 if len(gitCommit) == 0:
3184 print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3185 else:
3186 commitFound = True
3187 gitCommit = gitCommit.strip()
3189 if commitFound:
3190 # Convert from p4 time format
3191 try:
3192 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3193 except ValueError:
3194 print("Could not convert label time %s" % labelDetails['Update'])
3195 tmwhen = 1
3197 when = int(time.mktime(tmwhen))
3198 self.streamTag(stream, name, labelDetails, gitCommit, when)
3199 if verbose:
3200 print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3201 else:
3202 if verbose:
3203 print("Label %s has no changelists - possibly deleted?" % name)
3205 if not commitFound:
3206 # We can't import this label; don't try again as it will get very
3207 # expensive repeatedly fetching all the files for labels that will
3208 # never be imported. If the label is moved in the future, the
3209 # ignore will need to be removed manually.
3210 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3212 def guessProjectName(self):
3213 for p in self.depotPaths:
3214 if p.endswith("/"):
3215 p = p[:-1]
3216 p = p[p.strip().rfind("/") + 1:]
3217 if not p.endswith("/"):
3218 p += "/"
3219 return p
3221 def getBranchMapping(self):
3222 lostAndFoundBranches = set()
3224 user = gitConfig("git-p4.branchUser")
3225 if len(user) > 0:
3226 command = "branches -u %s" % user
3227 else:
3228 command = "branches"
3230 for info in p4CmdList(command):
3231 details = p4Cmd(["branch", "-o", info["branch"]])
3232 viewIdx = 0
3233 while "View%s" % viewIdx in details:
3234 paths = details["View%s" % viewIdx].split(" ")
3235 viewIdx = viewIdx + 1
3236 # require standard //depot/foo/... //depot/bar/... mapping
3237 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3238 continue
3239 source = paths[0]
3240 destination = paths[1]
3241 ## HACK
3242 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3243 source = source[len(self.depotPaths[0]):-4]
3244 destination = destination[len(self.depotPaths[0]):-4]
3246 if destination in self.knownBranches:
3247 if not self.silent:
3248 print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3249 print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3250 continue
3252 self.knownBranches[destination] = source
3254 lostAndFoundBranches.discard(destination)
3256 if source not in self.knownBranches:
3257 lostAndFoundBranches.add(source)
3259 # Perforce does not strictly require branches to be defined, so we also
3260 # check git config for a branch list.
3262 # Example of branch definition in git config file:
3263 # [git-p4]
3264 # branchList=main:branchA
3265 # branchList=main:branchB
3266 # branchList=branchA:branchC
3267 configBranches = gitConfigList("git-p4.branchList")
3268 for branch in configBranches:
3269 if branch:
3270 (source, destination) = branch.split(":")
3271 self.knownBranches[destination] = source
3273 lostAndFoundBranches.discard(destination)
3275 if source not in self.knownBranches:
3276 lostAndFoundBranches.add(source)
3279 for branch in lostAndFoundBranches:
3280 self.knownBranches[branch] = branch
3282 def getBranchMappingFromGitBranches(self):
3283 branches = p4BranchesInGit(self.importIntoRemotes)
3284 for branch in branches.keys():
3285 if branch == "master":
3286 branch = "main"
3287 else:
3288 branch = branch[len(self.projectName):]
3289 self.knownBranches[branch] = branch
3291 def updateOptionDict(self, d):
3292 option_keys = {}
3293 if self.keepRepoPath:
3294 option_keys['keepRepoPath'] = 1
3296 d["options"] = ' '.join(sorted(option_keys.keys()))
3298 def readOptions(self, d):
3299 self.keepRepoPath = ('options' in d
3300 and ('keepRepoPath' in d['options']))
3302 def gitRefForBranch(self, branch):
3303 if branch == "main":
3304 return self.refPrefix + "master"
3306 if len(branch) <= 0:
3307 return branch
3309 return self.refPrefix + self.projectName + branch
3311 def gitCommitByP4Change(self, ref, change):
3312 if self.verbose:
3313 print("looking in ref " + ref + " for change %s using bisect..." % change)
3315 earliestCommit = ""
3316 latestCommit = parseRevision(ref)
3318 while True:
3319 if self.verbose:
3320 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3321 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3322 if len(next) == 0:
3323 if self.verbose:
3324 print("argh")
3325 return ""
3326 log = extractLogMessageFromGitCommit(next)
3327 settings = extractSettingsGitLog(log)
3328 currentChange = int(settings['change'])
3329 if self.verbose:
3330 print("current change %s" % currentChange)
3332 if currentChange == change:
3333 if self.verbose:
3334 print("found %s" % next)
3335 return next
3337 if currentChange < change:
3338 earliestCommit = "^%s" % next
3339 else:
3340 latestCommit = "%s" % next
3342 return ""
3344 def importNewBranch(self, branch, maxChange):
3345 # make fast-import flush all changes to disk and update the refs using the checkpoint
3346 # command so that we can try to find the branch parent in the git history
3347 self.gitStream.write("checkpoint\n\n");
3348 self.gitStream.flush();
3349 branchPrefix = self.depotPaths[0] + branch + "/"
3350 range = "@1,%s" % maxChange
3351 #print "prefix" + branchPrefix
3352 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3353 if len(changes) <= 0:
3354 return False
3355 firstChange = changes[0]
3356 #print "first change in branch: %s" % firstChange
3357 sourceBranch = self.knownBranches[branch]
3358 sourceDepotPath = self.depotPaths[0] + sourceBranch
3359 sourceRef = self.gitRefForBranch(sourceBranch)
3360 #print "source " + sourceBranch
3362 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3363 #print "branch parent: %s" % branchParentChange
3364 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3365 if len(gitParent) > 0:
3366 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3367 #print "parent git commit: %s" % gitParent
3369 self.importChanges(changes)
3370 return True
3372 def searchParent(self, parent, branch, target):
3373 parentFound = False
3374 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3375 "--no-merges", parent]):
3376 blob = blob.strip()
3377 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3378 parentFound = True
3379 if self.verbose:
3380 print("Found parent of %s in commit %s" % (branch, blob))
3381 break
3382 if parentFound:
3383 return blob
3384 else:
3385 return None
3387 def importChanges(self, changes, shelved=False, origin_revision=0):
3388 cnt = 1
3389 for change in changes:
3390 description = p4_describe(change, shelved)
3391 self.updateOptionDict(description)
3393 if not self.silent:
3394 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3395 sys.stdout.flush()
3396 cnt = cnt + 1
3398 try:
3399 if self.detectBranches:
3400 branches = self.splitFilesIntoBranches(description)
3401 for branch in branches.keys():
3402 ## HACK --hwn
3403 branchPrefix = self.depotPaths[0] + branch + "/"
3404 self.branchPrefixes = [ branchPrefix ]
3406 parent = ""
3408 filesForCommit = branches[branch]
3410 if self.verbose:
3411 print("branch is %s" % branch)
3413 self.updatedBranches.add(branch)
3415 if branch not in self.createdBranches:
3416 self.createdBranches.add(branch)
3417 parent = self.knownBranches[branch]
3418 if parent == branch:
3419 parent = ""
3420 else:
3421 fullBranch = self.projectName + branch
3422 if fullBranch not in self.p4BranchesInGit:
3423 if not self.silent:
3424 print("\n Importing new branch %s" % fullBranch);
3425 if self.importNewBranch(branch, change - 1):
3426 parent = ""
3427 self.p4BranchesInGit.append(fullBranch)
3428 if not self.silent:
3429 print("\n Resuming with change %s" % change);
3431 if self.verbose:
3432 print("parent determined through known branches: %s" % parent)
3434 branch = self.gitRefForBranch(branch)
3435 parent = self.gitRefForBranch(parent)
3437 if self.verbose:
3438 print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3440 if len(parent) == 0 and branch in self.initialParents:
3441 parent = self.initialParents[branch]
3442 del self.initialParents[branch]
3444 blob = None
3445 if len(parent) > 0:
3446 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3447 if self.verbose:
3448 print("Creating temporary branch: " + tempBranch)
3449 self.commit(description, filesForCommit, tempBranch)
3450 self.tempBranches.append(tempBranch)
3451 self.checkpoint()
3452 blob = self.searchParent(parent, branch, tempBranch)
3453 if blob:
3454 self.commit(description, filesForCommit, branch, blob)
3455 else:
3456 if self.verbose:
3457 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3458 self.commit(description, filesForCommit, branch, parent)
3459 else:
3460 files = self.extractFilesFromCommit(description, shelved, change, origin_revision)
3461 self.commit(description, files, self.branch,
3462 self.initialParent)
3463 # only needed once, to connect to the previous commit
3464 self.initialParent = ""
3465 except IOError:
3466 print(self.gitError.read())
3467 sys.exit(1)
3469 def sync_origin_only(self):
3470 if self.syncWithOrigin:
3471 self.hasOrigin = originP4BranchesExist()
3472 if self.hasOrigin:
3473 if not self.silent:
3474 print('Syncing with origin first, using "git fetch origin"')
3475 system("git fetch origin")
3477 def importHeadRevision(self, revision):
3478 print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3480 details = {}
3481 details["user"] = "git perforce import user"
3482 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3483 % (' '.join(self.depotPaths), revision))
3484 details["change"] = revision
3485 newestRevision = 0
3487 fileCnt = 0
3488 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3490 for info in p4CmdList(["files"] + fileArgs):
3492 if 'code' in info and info['code'] == 'error':
3493 sys.stderr.write("p4 returned an error: %s\n"
3494 % info['data'])
3495 if info['data'].find("must refer to client") >= 0:
3496 sys.stderr.write("This particular p4 error is misleading.\n")
3497 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3498 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3499 sys.exit(1)
3500 if 'p4ExitCode' in info:
3501 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3502 sys.exit(1)
3505 change = int(info["change"])
3506 if change > newestRevision:
3507 newestRevision = change
3509 if info["action"] in self.delete_actions:
3510 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3511 #fileCnt = fileCnt + 1
3512 continue
3514 for prop in ["depotFile", "rev", "action", "type" ]:
3515 details["%s%s" % (prop, fileCnt)] = info[prop]
3517 fileCnt = fileCnt + 1
3519 details["change"] = newestRevision
3521 # Use time from top-most change so that all git p4 clones of
3522 # the same p4 repo have the same commit SHA1s.
3523 res = p4_describe(newestRevision)
3524 details["time"] = res["time"]
3526 self.updateOptionDict(details)
3527 try:
3528 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3529 except IOError:
3530 print("IO error with git fast-import. Is your git version recent enough?")
3531 print(self.gitError.read())
3533 def openStreams(self):
3534 self.importProcess = subprocess.Popen(["git", "fast-import"],
3535 stdin=subprocess.PIPE,
3536 stdout=subprocess.PIPE,
3537 stderr=subprocess.PIPE);
3538 self.gitOutput = self.importProcess.stdout
3539 self.gitStream = self.importProcess.stdin
3540 self.gitError = self.importProcess.stderr
3542 def closeStreams(self):
3543 self.gitStream.close()
3544 if self.importProcess.wait() != 0:
3545 die("fast-import failed: %s" % self.gitError.read())
3546 self.gitOutput.close()
3547 self.gitError.close()
3549 def run(self, args):
3550 if self.importIntoRemotes:
3551 self.refPrefix = "refs/remotes/p4/"
3552 else:
3553 self.refPrefix = "refs/heads/p4/"
3555 self.sync_origin_only()
3557 branch_arg_given = bool(self.branch)
3558 if len(self.branch) == 0:
3559 self.branch = self.refPrefix + "master"
3560 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3561 system("git update-ref %s refs/heads/p4" % self.branch)
3562 system("git branch -D p4")
3564 # accept either the command-line option, or the configuration variable
3565 if self.useClientSpec:
3566 # will use this after clone to set the variable
3567 self.useClientSpec_from_options = True
3568 else:
3569 if gitConfigBool("git-p4.useclientspec"):
3570 self.useClientSpec = True
3571 if self.useClientSpec:
3572 self.clientSpecDirs = getClientSpec()
3574 # TODO: should always look at previous commits,
3575 # merge with previous imports, if possible.
3576 if args == []:
3577 if self.hasOrigin:
3578 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3580 # branches holds mapping from branch name to sha1
3581 branches = p4BranchesInGit(self.importIntoRemotes)
3583 # restrict to just this one, disabling detect-branches
3584 if branch_arg_given:
3585 short = self.branch.split("/")[-1]
3586 if short in branches:
3587 self.p4BranchesInGit = [ short ]
3588 else:
3589 self.p4BranchesInGit = branches.keys()
3591 if len(self.p4BranchesInGit) > 1:
3592 if not self.silent:
3593 print("Importing from/into multiple branches")
3594 self.detectBranches = True
3595 for branch in branches.keys():
3596 self.initialParents[self.refPrefix + branch] = \
3597 branches[branch]
3599 if self.verbose:
3600 print("branches: %s" % self.p4BranchesInGit)
3602 p4Change = 0
3603 for branch in self.p4BranchesInGit:
3604 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3606 settings = extractSettingsGitLog(logMsg)
3608 self.readOptions(settings)
3609 if ('depot-paths' in settings
3610 and 'change' in settings):
3611 change = int(settings['change']) + 1
3612 p4Change = max(p4Change, change)
3614 depotPaths = sorted(settings['depot-paths'])
3615 if self.previousDepotPaths == []:
3616 self.previousDepotPaths = depotPaths
3617 else:
3618 paths = []
3619 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3620 prev_list = prev.split("/")
3621 cur_list = cur.split("/")
3622 for i in range(0, min(len(cur_list), len(prev_list))):
3623 if cur_list[i] != prev_list[i]:
3624 i = i - 1
3625 break
3627 paths.append ("/".join(cur_list[:i + 1]))
3629 self.previousDepotPaths = paths
3631 if p4Change > 0:
3632 self.depotPaths = sorted(self.previousDepotPaths)
3633 self.changeRange = "@%s,#head" % p4Change
3634 if not self.silent and not self.detectBranches:
3635 print("Performing incremental import into %s git branch" % self.branch)
3637 # accept multiple ref name abbreviations:
3638 # refs/foo/bar/branch -> use it exactly
3639 # p4/branch -> prepend refs/remotes/ or refs/heads/
3640 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3641 if not self.branch.startswith("refs/"):
3642 if self.importIntoRemotes:
3643 prepend = "refs/remotes/"
3644 else:
3645 prepend = "refs/heads/"
3646 if not self.branch.startswith("p4/"):
3647 prepend += "p4/"
3648 self.branch = prepend + self.branch
3650 if len(args) == 0 and self.depotPaths:
3651 if not self.silent:
3652 print("Depot paths: %s" % ' '.join(self.depotPaths))
3653 else:
3654 if self.depotPaths and self.depotPaths != args:
3655 print("previous import used depot path %s and now %s was specified. "
3656 "This doesn't work!" % (' '.join (self.depotPaths),
3657 ' '.join (args)))
3658 sys.exit(1)
3660 self.depotPaths = sorted(args)
3662 revision = ""
3663 self.users = {}
3665 # Make sure no revision specifiers are used when --changesfile
3666 # is specified.
3667 bad_changesfile = False
3668 if len(self.changesFile) > 0:
3669 for p in self.depotPaths:
3670 if p.find("@") >= 0 or p.find("#") >= 0:
3671 bad_changesfile = True
3672 break
3673 if bad_changesfile:
3674 die("Option --changesfile is incompatible with revision specifiers")
3676 newPaths = []
3677 for p in self.depotPaths:
3678 if p.find("@") != -1:
3679 atIdx = p.index("@")
3680 self.changeRange = p[atIdx:]
3681 if self.changeRange == "@all":
3682 self.changeRange = ""
3683 elif ',' not in self.changeRange:
3684 revision = self.changeRange
3685 self.changeRange = ""
3686 p = p[:atIdx]
3687 elif p.find("#") != -1:
3688 hashIdx = p.index("#")
3689 revision = p[hashIdx:]
3690 p = p[:hashIdx]
3691 elif self.previousDepotPaths == []:
3692 # pay attention to changesfile, if given, else import
3693 # the entire p4 tree at the head revision
3694 if len(self.changesFile) == 0:
3695 revision = "#head"
3697 p = re.sub ("\.\.\.$", "", p)
3698 if not p.endswith("/"):
3699 p += "/"
3701 newPaths.append(p)
3703 self.depotPaths = newPaths
3705 # --detect-branches may change this for each branch
3706 self.branchPrefixes = self.depotPaths
3708 self.loadUserMapFromCache()
3709 self.labels = {}
3710 if self.detectLabels:
3711 self.getLabels();
3713 if self.detectBranches:
3714 ## FIXME - what's a P4 projectName ?
3715 self.projectName = self.guessProjectName()
3717 if self.hasOrigin:
3718 self.getBranchMappingFromGitBranches()
3719 else:
3720 self.getBranchMapping()
3721 if self.verbose:
3722 print("p4-git branches: %s" % self.p4BranchesInGit)
3723 print("initial parents: %s" % self.initialParents)
3724 for b in self.p4BranchesInGit:
3725 if b != "master":
3727 ## FIXME
3728 b = b[len(self.projectName):]
3729 self.createdBranches.add(b)
3731 self.openStreams()
3733 if revision:
3734 self.importHeadRevision(revision)
3735 else:
3736 changes = []
3738 if len(self.changesFile) > 0:
3739 output = open(self.changesFile).readlines()
3740 changeSet = set()
3741 for line in output:
3742 changeSet.add(int(line))
3744 for change in changeSet:
3745 changes.append(change)
3747 changes.sort()
3748 else:
3749 # catch "git p4 sync" with no new branches, in a repo that
3750 # does not have any existing p4 branches
3751 if len(args) == 0:
3752 if not self.p4BranchesInGit:
3753 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3755 # The default branch is master, unless --branch is used to
3756 # specify something else. Make sure it exists, or complain
3757 # nicely about how to use --branch.
3758 if not self.detectBranches:
3759 if not branch_exists(self.branch):
3760 if branch_arg_given:
3761 die("Error: branch %s does not exist." % self.branch)
3762 else:
3763 die("Error: no branch %s; perhaps specify one with --branch." %
3764 self.branch)
3766 if self.verbose:
3767 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3768 self.changeRange))
3769 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3771 if len(self.maxChanges) > 0:
3772 changes = changes[:min(int(self.maxChanges), len(changes))]
3774 if len(changes) == 0:
3775 if not self.silent:
3776 print("No changes to import!")
3777 else:
3778 if not self.silent and not self.detectBranches:
3779 print("Import destination: %s" % self.branch)
3781 self.updatedBranches = set()
3783 if not self.detectBranches:
3784 if args:
3785 # start a new branch
3786 self.initialParent = ""
3787 else:
3788 # build on a previous revision
3789 self.initialParent = parseRevision(self.branch)
3791 self.importChanges(changes)
3793 if not self.silent:
3794 print("")
3795 if len(self.updatedBranches) > 0:
3796 sys.stdout.write("Updated branches: ")
3797 for b in self.updatedBranches:
3798 sys.stdout.write("%s " % b)
3799 sys.stdout.write("\n")
3801 if gitConfigBool("git-p4.importLabels"):
3802 self.importLabels = True
3804 if self.importLabels:
3805 p4Labels = getP4Labels(self.depotPaths)
3806 gitTags = getGitTags()
3808 missingP4Labels = p4Labels - gitTags
3809 self.importP4Labels(self.gitStream, missingP4Labels)
3811 self.closeStreams()
3813 # Cleanup temporary branches created during import
3814 if self.tempBranches != []:
3815 for branch in self.tempBranches:
3816 read_pipe("git update-ref -d %s" % branch)
3817 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3819 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3820 # a convenient shortcut refname "p4".
3821 if self.importIntoRemotes:
3822 head_ref = self.refPrefix + "HEAD"
3823 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3824 system(["git", "symbolic-ref", head_ref, self.branch])
3826 return True
3828 class P4Rebase(Command):
3829 def __init__(self):
3830 Command.__init__(self)
3831 self.options = [
3832 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3834 self.importLabels = False
3835 self.description = ("Fetches the latest revision from perforce and "
3836 + "rebases the current work (branch) against it")
3838 def run(self, args):
3839 sync = P4Sync()
3840 sync.importLabels = self.importLabels
3841 sync.run([])
3843 return self.rebase()
3845 def rebase(self):
3846 if os.system("git update-index --refresh") != 0:
3847 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3848 if len(read_pipe("git diff-index HEAD --")) > 0:
3849 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3851 [upstream, settings] = findUpstreamBranchPoint()
3852 if len(upstream) == 0:
3853 die("Cannot find upstream branchpoint for rebase")
3855 # the branchpoint may be p4/foo~3, so strip off the parent
3856 upstream = re.sub("~[0-9]+$", "", upstream)
3858 print("Rebasing the current branch onto %s" % upstream)
3859 oldHead = read_pipe("git rev-parse HEAD").strip()
3860 system("git rebase %s" % upstream)
3861 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3862 return True
3864 class P4Clone(P4Sync):
3865 def __init__(self):
3866 P4Sync.__init__(self)
3867 self.description = "Creates a new git repository and imports from Perforce into it"
3868 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3869 self.options += [
3870 optparse.make_option("--destination", dest="cloneDestination",
3871 action='store', default=None,
3872 help="where to leave result of the clone"),
3873 optparse.make_option("--bare", dest="cloneBare",
3874 action="store_true", default=False),
3876 self.cloneDestination = None
3877 self.needsGit = False
3878 self.cloneBare = False
3880 def defaultDestination(self, args):
3881 ## TODO: use common prefix of args?
3882 depotPath = args[0]
3883 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3884 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3885 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3886 depotDir = re.sub(r"/$", "", depotDir)
3887 return os.path.split(depotDir)[1]
3889 def run(self, args):
3890 if len(args) < 1:
3891 return False
3893 if self.keepRepoPath and not self.cloneDestination:
3894 sys.stderr.write("Must specify destination for --keep-path\n")
3895 sys.exit(1)
3897 depotPaths = args
3899 if not self.cloneDestination and len(depotPaths) > 1:
3900 self.cloneDestination = depotPaths[-1]
3901 depotPaths = depotPaths[:-1]
3903 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3904 for p in depotPaths:
3905 if not p.startswith("//"):
3906 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3907 return False
3909 if not self.cloneDestination:
3910 self.cloneDestination = self.defaultDestination(args)
3912 print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
3914 if not os.path.exists(self.cloneDestination):
3915 os.makedirs(self.cloneDestination)
3916 chdir(self.cloneDestination)
3918 init_cmd = [ "git", "init" ]
3919 if self.cloneBare:
3920 init_cmd.append("--bare")
3921 retcode = subprocess.call(init_cmd)
3922 if retcode:
3923 raise CalledProcessError(retcode, init_cmd)
3925 if not P4Sync.run(self, depotPaths):
3926 return False
3928 # create a master branch and check out a work tree
3929 if gitBranchExists(self.branch):
3930 system([ "git", "branch", "master", self.branch ])
3931 if not self.cloneBare:
3932 system([ "git", "checkout", "-f" ])
3933 else:
3934 print('Not checking out any branch, use ' \
3935 '"git checkout -q -b master <branch>"')
3937 # auto-set this variable if invoked with --use-client-spec
3938 if self.useClientSpec_from_options:
3939 system("git config --bool git-p4.useclientspec true")
3941 return True
3943 class P4Unshelve(Command):
3944 def __init__(self):
3945 Command.__init__(self)
3946 self.options = []
3947 self.origin = "HEAD"
3948 self.description = "Unshelve a P4 changelist into a git commit"
3949 self.usage = "usage: %prog [options] changelist"
3950 self.options += [
3951 optparse.make_option("--origin", dest="origin",
3952 help="Use this base revision instead of the default (%s)" % self.origin),
3954 self.verbose = False
3955 self.noCommit = False
3956 self.destbranch = "refs/remotes/p4/unshelved"
3958 def renameBranch(self, branch_name):
3959 """ Rename the existing branch to branch_name.N
3962 found = True
3963 for i in range(0,1000):
3964 backup_branch_name = "{0}.{1}".format(branch_name, i)
3965 if not gitBranchExists(backup_branch_name):
3966 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3967 gitDeleteRef(branch_name)
3968 found = True
3969 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3970 break
3972 if not found:
3973 sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3975 def findLastP4Revision(self, starting_point):
3976 """ Look back from starting_point for the first commit created by git-p4
3977 to find the P4 commit we are based on, and the depot-paths.
3980 for parent in (range(65535)):
3981 log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3982 settings = extractSettingsGitLog(log)
3983 if 'change' in settings:
3984 return settings
3986 sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3988 def run(self, args):
3989 if len(args) != 1:
3990 return False
3992 if not gitBranchExists(self.origin):
3993 sys.exit("origin branch {0} does not exist".format(self.origin))
3995 sync = P4Sync()
3996 changes = args
3997 sync.initialParent = self.origin
3999 # use the first change in the list to construct the branch to unshelve into
4000 change = changes[0]
4002 # if the target branch already exists, rename it
4003 branch_name = "{0}/{1}".format(self.destbranch, change)
4004 if gitBranchExists(branch_name):
4005 self.renameBranch(branch_name)
4006 sync.branch = branch_name
4008 sync.verbose = self.verbose
4009 sync.suppress_meta_comment = True
4011 settings = self.findLastP4Revision(self.origin)
4012 origin_revision = settings['change']
4013 sync.depotPaths = settings['depot-paths']
4014 sync.branchPrefixes = sync.depotPaths
4016 sync.openStreams()
4017 sync.loadUserMapFromCache()
4018 sync.silent = True
4019 sync.importChanges(changes, shelved=True, origin_revision=origin_revision)
4020 sync.closeStreams()
4022 print("unshelved changelist {0} into {1}".format(change, branch_name))
4024 return True
4026 class P4Branches(Command):
4027 def __init__(self):
4028 Command.__init__(self)
4029 self.options = [ ]
4030 self.description = ("Shows the git branches that hold imports and their "
4031 + "corresponding perforce depot paths")
4032 self.verbose = False
4034 def run(self, args):
4035 if originP4BranchesExist():
4036 createOrUpdateBranchesFromOrigin()
4038 cmdline = "git rev-parse --symbolic "
4039 cmdline += " --remotes"
4041 for line in read_pipe_lines(cmdline):
4042 line = line.strip()
4044 if not line.startswith('p4/') or line == "p4/HEAD":
4045 continue
4046 branch = line
4048 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4049 settings = extractSettingsGitLog(log)
4051 print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4052 return True
4054 class HelpFormatter(optparse.IndentedHelpFormatter):
4055 def __init__(self):
4056 optparse.IndentedHelpFormatter.__init__(self)
4058 def format_description(self, description):
4059 if description:
4060 return description + "\n"
4061 else:
4062 return ""
4064 def printUsage(commands):
4065 print("usage: %s <command> [options]" % sys.argv[0])
4066 print("")
4067 print("valid commands: %s" % ", ".join(commands))
4068 print("")
4069 print("Try %s <command> --help for command specific help." % sys.argv[0])
4070 print("")
4072 commands = {
4073 "debug" : P4Debug,
4074 "submit" : P4Submit,
4075 "commit" : P4Submit,
4076 "sync" : P4Sync,
4077 "rebase" : P4Rebase,
4078 "clone" : P4Clone,
4079 "rollback" : P4RollBack,
4080 "branches" : P4Branches,
4081 "unshelve" : P4Unshelve,
4085 def main():
4086 if len(sys.argv[1:]) == 0:
4087 printUsage(commands.keys())
4088 sys.exit(2)
4090 cmdName = sys.argv[1]
4091 try:
4092 klass = commands[cmdName]
4093 cmd = klass()
4094 except KeyError:
4095 print("unknown command %s" % cmdName)
4096 print("")
4097 printUsage(commands.keys())
4098 sys.exit(2)
4100 options = cmd.options
4101 cmd.gitdir = os.environ.get("GIT_DIR", None)
4103 args = sys.argv[2:]
4105 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4106 if cmd.needsGit:
4107 options.append(optparse.make_option("--git-dir", dest="gitdir"))
4109 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4110 options,
4111 description = cmd.description,
4112 formatter = HelpFormatter())
4114 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4115 global verbose
4116 verbose = cmd.verbose
4117 if cmd.needsGit:
4118 if cmd.gitdir == None:
4119 cmd.gitdir = os.path.abspath(".git")
4120 if not isValidGitDir(cmd.gitdir):
4121 # "rev-parse --git-dir" without arguments will try $PWD/.git
4122 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4123 if os.path.exists(cmd.gitdir):
4124 cdup = read_pipe("git rev-parse --show-cdup").strip()
4125 if len(cdup) > 0:
4126 chdir(cdup);
4128 if not isValidGitDir(cmd.gitdir):
4129 if isValidGitDir(cmd.gitdir + "/.git"):
4130 cmd.gitdir += "/.git"
4131 else:
4132 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4134 # so git commands invoked from the P4 workspace will succeed
4135 os.environ["GIT_DIR"] = cmd.gitdir
4137 if not cmd.run(args):
4138 parser.print_help()
4139 sys.exit(2)
4142 if __name__ == '__main__':
4143 main()