Git 2.35-rc1
[alt-git.git] / git-p4.py
blobcb37545455e9d8b782c078582a7e5a8b902af845
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 # pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except
11 # pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary
12 # pylint: disable=wrong-import-order,unused-import,too-few-public-methods
13 # pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals
14 # pylint: disable=line-too-long,bad-whitespace,superfluous-parens
15 # pylint: disable=too-many-statements,too-many-instance-attributes
16 # pylint: disable=too-many-branches,too-many-nested-blocks
18 import sys
19 if sys.version_info.major < 3 and sys.version_info.minor < 7:
20 sys.stderr.write("git-p4: requires Python 2.7 or later.\n")
21 sys.exit(1)
22 import os
23 import optparse
24 import functools
25 import marshal
26 import subprocess
27 import tempfile
28 import time
29 import platform
30 import re
31 import shutil
32 import stat
33 import zipfile
34 import zlib
35 import ctypes
36 import errno
37 import glob
39 # On python2.7 where raw_input() and input() are both availble,
40 # we want raw_input's semantics, but aliased to input for python3
41 # compatibility
42 # support basestring in python3
43 try:
44 if raw_input and input:
45 input = raw_input
46 except:
47 pass
49 verbose = False
51 # Only labels/tags matching this will be imported/exported
52 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
54 # The block size is reduced automatically if required
55 defaultBlockSize = 1<<20
57 p4_access_checked = False
59 re_ko_keywords = re.compile(br'\$(Id|Header)(:[^$\n]+)?\$')
60 re_k_keywords = re.compile(br'\$(Id|Header|Author|Date|DateTime|Change|File|Revision)(:[^$\n]+)?\$')
62 def format_size_human_readable(num):
63 """ Returns a number of units (typically bytes) formatted as a human-readable
64 string.
65 """
66 if num < 1024:
67 return '{:d} B'.format(num)
68 for unit in ["Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
69 num /= 1024.0
70 if num < 1024.0:
71 return "{:3.1f} {}B".format(num, unit)
72 return "{:.1f} YiB".format(num)
74 def p4_build_cmd(cmd):
75 """Build a suitable p4 command line.
77 This consolidates building and returning a p4 command line into one
78 location. It means that hooking into the environment, or other configuration
79 can be done more easily.
80 """
81 real_cmd = ["p4"]
83 user = gitConfig("git-p4.user")
84 if len(user) > 0:
85 real_cmd += ["-u",user]
87 password = gitConfig("git-p4.password")
88 if len(password) > 0:
89 real_cmd += ["-P", password]
91 port = gitConfig("git-p4.port")
92 if len(port) > 0:
93 real_cmd += ["-p", port]
95 host = gitConfig("git-p4.host")
96 if len(host) > 0:
97 real_cmd += ["-H", host]
99 client = gitConfig("git-p4.client")
100 if len(client) > 0:
101 real_cmd += ["-c", client]
103 retries = gitConfigInt("git-p4.retries")
104 if retries is None:
105 # Perform 3 retries by default
106 retries = 3
107 if retries > 0:
108 # Provide a way to not pass this option by setting git-p4.retries to 0
109 real_cmd += ["-r", str(retries)]
111 if not isinstance(cmd, list):
112 real_cmd = ' '.join(real_cmd) + ' ' + cmd
113 else:
114 real_cmd += cmd
116 # now check that we can actually talk to the server
117 global p4_access_checked
118 if not p4_access_checked:
119 p4_access_checked = True # suppress access checks in p4_check_access itself
120 p4_check_access()
122 return real_cmd
124 def git_dir(path):
125 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
126 This won't automatically add ".git" to a directory.
128 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
129 if not d or len(d) == 0:
130 return None
131 else:
132 return d
134 def chdir(path, is_client_path=False):
135 """Do chdir to the given path, and set the PWD environment
136 variable for use by P4. It does not look at getcwd() output.
137 Since we're not using the shell, it is necessary to set the
138 PWD environment variable explicitly.
140 Normally, expand the path to force it to be absolute. This
141 addresses the use of relative path names inside P4 settings,
142 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
143 as given; it looks for .p4config using PWD.
145 If is_client_path, the path was handed to us directly by p4,
146 and may be a symbolic link. Do not call os.getcwd() in this
147 case, because it will cause p4 to think that PWD is not inside
148 the client path.
151 os.chdir(path)
152 if not is_client_path:
153 path = os.getcwd()
154 os.environ['PWD'] = path
156 def calcDiskFree():
157 """Return free space in bytes on the disk of the given dirname."""
158 if platform.system() == 'Windows':
159 free_bytes = ctypes.c_ulonglong(0)
160 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
161 return free_bytes.value
162 else:
163 st = os.statvfs(os.getcwd())
164 return st.f_bavail * st.f_frsize
166 def die(msg):
167 """ Terminate execution. Make sure that any running child processes have been wait()ed for before
168 calling this.
170 if verbose:
171 raise Exception(msg)
172 else:
173 sys.stderr.write(msg + "\n")
174 sys.exit(1)
176 def prompt(prompt_text):
177 """ Prompt the user to choose one of the choices
179 Choices are identified in the prompt_text by square brackets around
180 a single letter option.
182 choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
183 while True:
184 sys.stderr.flush()
185 sys.stdout.write(prompt_text)
186 sys.stdout.flush()
187 response=sys.stdin.readline().strip().lower()
188 if not response:
189 continue
190 response = response[0]
191 if response in choices:
192 return response
194 # We need different encoding/decoding strategies for text data being passed
195 # around in pipes depending on python version
196 if bytes is not str:
197 # For python3, always encode and decode as appropriate
198 def decode_text_stream(s):
199 return s.decode() if isinstance(s, bytes) else s
200 def encode_text_stream(s):
201 return s.encode() if isinstance(s, str) else s
202 else:
203 # For python2.7, pass read strings as-is, but also allow writing unicode
204 def decode_text_stream(s):
205 return s
206 def encode_text_stream(s):
207 return s.encode('utf_8') if isinstance(s, unicode) else s
209 def decode_path(path):
210 """Decode a given string (bytes or otherwise) using configured path encoding options
212 encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'
213 if bytes is not str:
214 return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path
215 else:
216 try:
217 path.decode('ascii')
218 except:
219 path = path.decode(encoding, errors='replace')
220 if verbose:
221 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))
222 return path
224 def run_git_hook(cmd, param=[]):
225 """Execute a hook if the hook exists."""
226 if verbose:
227 sys.stderr.write("Looking for hook: %s\n" % cmd)
228 sys.stderr.flush()
230 hooks_path = gitConfig("core.hooksPath")
231 if len(hooks_path) <= 0:
232 hooks_path = os.path.join(os.environ["GIT_DIR"], "hooks")
234 if not isinstance(param, list):
235 param=[param]
237 # resolve hook file name, OS depdenent
238 hook_file = os.path.join(hooks_path, cmd)
239 if platform.system() == 'Windows':
240 if not os.path.isfile(hook_file):
241 # look for the file with an extension
242 files = glob.glob(hook_file + ".*")
243 if not files:
244 return True
245 files.sort()
246 hook_file = files.pop()
247 while hook_file.upper().endswith(".SAMPLE"):
248 # The file is a sample hook. We don't want it
249 if len(files) > 0:
250 hook_file = files.pop()
251 else:
252 return True
254 if not os.path.isfile(hook_file) or not os.access(hook_file, os.X_OK):
255 return True
257 return run_hook_command(hook_file, param) == 0
259 def run_hook_command(cmd, param):
260 """Executes a git hook command
261 cmd = the command line file to be executed. This can be
262 a file that is run by OS association.
264 param = a list of parameters to pass to the cmd command
266 On windows, the extension is checked to see if it should
267 be run with the Git for Windows Bash shell. If there
268 is no file extension, the file is deemed a bash shell
269 and will be handed off to sh.exe. Otherwise, Windows
270 will be called with the shell to handle the file assocation.
272 For non Windows operating systems, the file is called
273 as an executable.
275 cli = [cmd] + param
276 use_shell = False
277 if platform.system() == 'Windows':
278 (root,ext) = os.path.splitext(cmd)
279 if ext == "":
280 exe_path = os.environ.get("EXEPATH")
281 if exe_path is None:
282 exe_path = ""
283 else:
284 exe_path = os.path.join(exe_path, "bin")
285 cli = [os.path.join(exe_path, "SH.EXE")] + cli
286 else:
287 use_shell = True
288 return subprocess.call(cli, shell=use_shell)
291 def write_pipe(c, stdin):
292 if verbose:
293 sys.stderr.write('Writing pipe: %s\n' % str(c))
295 expand = not isinstance(c, list)
296 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
297 pipe = p.stdin
298 val = pipe.write(stdin)
299 pipe.close()
300 if p.wait():
301 die('Command failed: %s' % str(c))
303 return val
305 def p4_write_pipe(c, stdin):
306 real_cmd = p4_build_cmd(c)
307 if bytes is not str and isinstance(stdin, str):
308 stdin = encode_text_stream(stdin)
309 return write_pipe(real_cmd, stdin)
311 def read_pipe_full(c):
312 """ Read output from command. Returns a tuple
313 of the return status, stdout text and stderr
314 text.
316 if verbose:
317 sys.stderr.write('Reading pipe: %s\n' % str(c))
319 expand = not isinstance(c, list)
320 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
321 (out, err) = p.communicate()
322 return (p.returncode, out, decode_text_stream(err))
324 def read_pipe(c, ignore_error=False, raw=False):
325 """ Read output from command. Returns the output text on
326 success. On failure, terminates execution, unless
327 ignore_error is True, when it returns an empty string.
329 If raw is True, do not attempt to decode output text.
331 (retcode, out, err) = read_pipe_full(c)
332 if retcode != 0:
333 if ignore_error:
334 out = ""
335 else:
336 die('Command failed: %s\nError: %s' % (str(c), err))
337 if not raw:
338 out = decode_text_stream(out)
339 return out
341 def read_pipe_text(c):
342 """ Read output from a command with trailing whitespace stripped.
343 On error, returns None.
345 (retcode, out, err) = read_pipe_full(c)
346 if retcode != 0:
347 return None
348 else:
349 return decode_text_stream(out).rstrip()
351 def p4_read_pipe(c, ignore_error=False, raw=False):
352 real_cmd = p4_build_cmd(c)
353 return read_pipe(real_cmd, ignore_error, raw=raw)
355 def read_pipe_lines(c, raw=False):
356 if verbose:
357 sys.stderr.write('Reading pipe: %s\n' % str(c))
359 expand = not isinstance(c, list)
360 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
361 pipe = p.stdout
362 lines = pipe.readlines()
363 if not raw:
364 lines = [decode_text_stream(line) for line in lines]
365 if pipe.close() or p.wait():
366 die('Command failed: %s' % str(c))
367 return lines
369 def p4_read_pipe_lines(c):
370 """Specifically invoke p4 on the command supplied. """
371 real_cmd = p4_build_cmd(c)
372 return read_pipe_lines(real_cmd)
374 def p4_has_command(cmd):
375 """Ask p4 for help on this command. If it returns an error, the
376 command does not exist in this version of p4."""
377 real_cmd = p4_build_cmd(["help", cmd])
378 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
379 stderr=subprocess.PIPE)
380 p.communicate()
381 return p.returncode == 0
383 def p4_has_move_command():
384 """See if the move command exists, that it supports -k, and that
385 it has not been administratively disabled. The arguments
386 must be correct, but the filenames do not have to exist. Use
387 ones with wildcards so even if they exist, it will fail."""
389 if not p4_has_command("move"):
390 return False
391 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
392 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
393 (out, err) = p.communicate()
394 err = decode_text_stream(err)
395 # return code will be 1 in either case
396 if err.find("Invalid option") >= 0:
397 return False
398 if err.find("disabled") >= 0:
399 return False
400 # assume it failed because @... was invalid changelist
401 return True
403 def system(cmd, ignore_error=False):
404 expand = not isinstance(cmd, list)
405 if verbose:
406 sys.stderr.write("executing %s\n" % str(cmd))
407 retcode = subprocess.call(cmd, shell=expand)
408 if retcode and not ignore_error:
409 raise CalledProcessError(retcode, cmd)
411 return retcode
413 def p4_system(cmd):
414 """Specifically invoke p4 as the system command. """
415 real_cmd = p4_build_cmd(cmd)
416 expand = not isinstance(real_cmd, list)
417 retcode = subprocess.call(real_cmd, shell=expand)
418 if retcode:
419 raise CalledProcessError(retcode, real_cmd)
421 def die_bad_access(s):
422 die("failure accessing depot: {0}".format(s.rstrip()))
424 def p4_check_access(min_expiration=1):
425 """ Check if we can access Perforce - account still logged in
427 results = p4CmdList(["login", "-s"])
429 if len(results) == 0:
430 # should never get here: always get either some results, or a p4ExitCode
431 assert("could not parse response from perforce")
433 result = results[0]
435 if 'p4ExitCode' in result:
436 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
437 die_bad_access("could not run p4")
439 code = result.get("code")
440 if not code:
441 # we get here if we couldn't connect and there was nothing to unmarshal
442 die_bad_access("could not connect")
444 elif code == "stat":
445 expiry = result.get("TicketExpiration")
446 if expiry:
447 expiry = int(expiry)
448 if expiry > min_expiration:
449 # ok to carry on
450 return
451 else:
452 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
454 else:
455 # account without a timeout - all ok
456 return
458 elif code == "error":
459 data = result.get("data")
460 if data:
461 die_bad_access("p4 error: {0}".format(data))
462 else:
463 die_bad_access("unknown error")
464 elif code == "info":
465 return
466 else:
467 die_bad_access("unknown error code {0}".format(code))
469 _p4_version_string = None
470 def p4_version_string():
471 """Read the version string, showing just the last line, which
472 hopefully is the interesting version bit.
474 $ p4 -V
475 Perforce - The Fast Software Configuration Management System.
476 Copyright 1995-2011 Perforce Software. All rights reserved.
477 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
479 global _p4_version_string
480 if not _p4_version_string:
481 a = p4_read_pipe_lines(["-V"])
482 _p4_version_string = a[-1].rstrip()
483 return _p4_version_string
485 def p4_integrate(src, dest):
486 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
488 def p4_sync(f, *options):
489 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
491 def p4_add(f):
492 # forcibly add file names with wildcards
493 if wildcard_present(f):
494 p4_system(["add", "-f", f])
495 else:
496 p4_system(["add", f])
498 def p4_delete(f):
499 p4_system(["delete", wildcard_encode(f)])
501 def p4_edit(f, *options):
502 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
504 def p4_revert(f):
505 p4_system(["revert", wildcard_encode(f)])
507 def p4_reopen(type, f):
508 p4_system(["reopen", "-t", type, wildcard_encode(f)])
510 def p4_reopen_in_change(changelist, files):
511 cmd = ["reopen", "-c", str(changelist)] + files
512 p4_system(cmd)
514 def p4_move(src, dest):
515 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
517 def p4_last_change():
518 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
519 return int(results[0]['change'])
521 def p4_describe(change, shelved=False):
522 """Make sure it returns a valid result by checking for
523 the presence of field "time". Return a dict of the
524 results."""
526 cmd = ["describe", "-s"]
527 if shelved:
528 cmd += ["-S"]
529 cmd += [str(change)]
531 ds = p4CmdList(cmd, skip_info=True)
532 if len(ds) != 1:
533 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
535 d = ds[0]
537 if "p4ExitCode" in d:
538 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
539 str(d)))
540 if "code" in d:
541 if d["code"] == "error":
542 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
544 if "time" not in d:
545 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
547 return d
550 # Canonicalize the p4 type and return a tuple of the
551 # base type, plus any modifiers. See "p4 help filetypes"
552 # for a list and explanation.
554 def split_p4_type(p4type):
556 p4_filetypes_historical = {
557 "ctempobj": "binary+Sw",
558 "ctext": "text+C",
559 "cxtext": "text+Cx",
560 "ktext": "text+k",
561 "kxtext": "text+kx",
562 "ltext": "text+F",
563 "tempobj": "binary+FSw",
564 "ubinary": "binary+F",
565 "uresource": "resource+F",
566 "uxbinary": "binary+Fx",
567 "xbinary": "binary+x",
568 "xltext": "text+Fx",
569 "xtempobj": "binary+Swx",
570 "xtext": "text+x",
571 "xunicode": "unicode+x",
572 "xutf16": "utf16+x",
574 if p4type in p4_filetypes_historical:
575 p4type = p4_filetypes_historical[p4type]
576 mods = ""
577 s = p4type.split("+")
578 base = s[0]
579 mods = ""
580 if len(s) > 1:
581 mods = s[1]
582 return (base, mods)
585 # return the raw p4 type of a file (text, text+ko, etc)
587 def p4_type(f):
588 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
589 return results[0]['headType']
592 # Given a type base and modifier, return a regexp matching
593 # the keywords that can be expanded in the file
595 def p4_keywords_regexp_for_type(base, type_mods):
596 if base in ("text", "unicode", "binary"):
597 if "ko" in type_mods:
598 return re_ko_keywords
599 elif "k" in type_mods:
600 return re_k_keywords
601 else:
602 return None
603 else:
604 return None
607 # Given a file, return a regexp matching the possible
608 # RCS keywords that will be expanded, or None for files
609 # with kw expansion turned off.
611 def p4_keywords_regexp_for_file(file):
612 if not os.path.exists(file):
613 return None
614 else:
615 (type_base, type_mods) = split_p4_type(p4_type(file))
616 return p4_keywords_regexp_for_type(type_base, type_mods)
618 def setP4ExecBit(file, mode):
619 # Reopens an already open file and changes the execute bit to match
620 # the execute bit setting in the passed in mode.
622 p4Type = "+x"
624 if not isModeExec(mode):
625 p4Type = getP4OpenedType(file)
626 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
627 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
628 if p4Type[-1] == "+":
629 p4Type = p4Type[0:-1]
631 p4_reopen(p4Type, file)
633 def getP4OpenedType(file):
634 # Returns the perforce file type for the given file.
636 result = p4_read_pipe(["opened", wildcard_encode(file)])
637 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
638 if match:
639 return match.group(1)
640 else:
641 die("Could not determine file type for %s (result: '%s')" % (file, result))
643 # Return the set of all p4 labels
644 def getP4Labels(depotPaths):
645 labels = set()
646 if not isinstance(depotPaths, list):
647 depotPaths = [depotPaths]
649 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
650 label = l['label']
651 labels.add(label)
653 return labels
655 # Return the set of all git tags
656 def getGitTags():
657 gitTags = set()
658 for line in read_pipe_lines(["git", "tag"]):
659 tag = line.strip()
660 gitTags.add(tag)
661 return gitTags
663 _diff_tree_pattern = None
665 def parseDiffTreeEntry(entry):
666 """Parses a single diff tree entry into its component elements.
668 See git-diff-tree(1) manpage for details about the format of the diff
669 output. This method returns a dictionary with the following elements:
671 src_mode - The mode of the source file
672 dst_mode - The mode of the destination file
673 src_sha1 - The sha1 for the source file
674 dst_sha1 - The sha1 fr the destination file
675 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
676 status_score - The score for the status (applicable for 'C' and 'R'
677 statuses). This is None if there is no score.
678 src - The path for the source file.
679 dst - The path for the destination file. This is only present for
680 copy or renames. If it is not present, this is None.
682 If the pattern is not matched, None is returned."""
684 global _diff_tree_pattern
685 if not _diff_tree_pattern:
686 _diff_tree_pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
688 match = _diff_tree_pattern.match(entry)
689 if match:
690 return {
691 'src_mode': match.group(1),
692 'dst_mode': match.group(2),
693 'src_sha1': match.group(3),
694 'dst_sha1': match.group(4),
695 'status': match.group(5),
696 'status_score': match.group(6),
697 'src': match.group(7),
698 'dst': match.group(10)
700 return None
702 def isModeExec(mode):
703 # Returns True if the given git mode represents an executable file,
704 # otherwise False.
705 return mode[-3:] == "755"
707 class P4Exception(Exception):
708 """ Base class for exceptions from the p4 client """
709 def __init__(self, exit_code):
710 self.p4ExitCode = exit_code
712 class P4ServerException(P4Exception):
713 """ Base class for exceptions where we get some kind of marshalled up result from the server """
714 def __init__(self, exit_code, p4_result):
715 super(P4ServerException, self).__init__(exit_code)
716 self.p4_result = p4_result
717 self.code = p4_result[0]['code']
718 self.data = p4_result[0]['data']
720 class P4RequestSizeException(P4ServerException):
721 """ One of the maxresults or maxscanrows errors """
722 def __init__(self, exit_code, p4_result, limit):
723 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
724 self.limit = limit
726 class P4CommandException(P4Exception):
727 """ Something went wrong calling p4 which means we have to give up """
728 def __init__(self, msg):
729 self.msg = msg
731 def __str__(self):
732 return self.msg
734 def isModeExecChanged(src_mode, dst_mode):
735 return isModeExec(src_mode) != isModeExec(dst_mode)
737 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
738 errors_as_exceptions=False):
740 if not isinstance(cmd, list):
741 cmd = "-G " + cmd
742 expand = True
743 else:
744 cmd = ["-G"] + cmd
745 expand = False
747 cmd = p4_build_cmd(cmd)
748 if verbose:
749 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
751 # Use a temporary file to avoid deadlocks without
752 # subprocess.communicate(), which would put another copy
753 # of stdout into memory.
754 stdin_file = None
755 if stdin is not None:
756 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
757 if not isinstance(stdin, list):
758 stdin_file.write(stdin)
759 else:
760 for i in stdin:
761 stdin_file.write(encode_text_stream(i))
762 stdin_file.write(b'\n')
763 stdin_file.flush()
764 stdin_file.seek(0)
766 p4 = subprocess.Popen(cmd,
767 shell=expand,
768 stdin=stdin_file,
769 stdout=subprocess.PIPE)
771 result = []
772 try:
773 while True:
774 entry = marshal.load(p4.stdout)
775 if bytes is not str:
776 # Decode unmarshalled dict to use str keys and values, except for:
777 # - `data` which may contain arbitrary binary data
778 # - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
779 decoded_entry = {}
780 for key, value in entry.items():
781 key = key.decode()
782 if isinstance(value, bytes) and not (key in ('data', 'path', 'clientFile') or key.startswith('depotFile')):
783 value = value.decode()
784 decoded_entry[key] = value
785 # Parse out data if it's an error response
786 if decoded_entry.get('code') == 'error' and 'data' in decoded_entry:
787 decoded_entry['data'] = decoded_entry['data'].decode()
788 entry = decoded_entry
789 if skip_info:
790 if 'code' in entry and entry['code'] == 'info':
791 continue
792 if cb is not None:
793 cb(entry)
794 else:
795 result.append(entry)
796 except EOFError:
797 pass
798 exitCode = p4.wait()
799 if exitCode != 0:
800 if errors_as_exceptions:
801 if len(result) > 0:
802 data = result[0].get('data')
803 if data:
804 m = re.search('Too many rows scanned \(over (\d+)\)', data)
805 if not m:
806 m = re.search('Request too large \(over (\d+)\)', data)
808 if m:
809 limit = int(m.group(1))
810 raise P4RequestSizeException(exitCode, result, limit)
812 raise P4ServerException(exitCode, result)
813 else:
814 raise P4Exception(exitCode)
815 else:
816 entry = {}
817 entry["p4ExitCode"] = exitCode
818 result.append(entry)
820 return result
822 def p4Cmd(cmd):
823 list = p4CmdList(cmd)
824 result = {}
825 for entry in list:
826 result.update(entry)
827 return result;
829 def p4Where(depotPath):
830 if not depotPath.endswith("/"):
831 depotPath += "/"
832 depotPathLong = depotPath + "..."
833 outputList = p4CmdList(["where", depotPathLong])
834 output = None
835 for entry in outputList:
836 if "depotFile" in entry:
837 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
838 # The base path always ends with "/...".
839 entry_path = decode_path(entry['depotFile'])
840 if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":
841 output = entry
842 break
843 elif "data" in entry:
844 data = entry.get("data")
845 space = data.find(" ")
846 if data[:space] == depotPath:
847 output = entry
848 break
849 if output == None:
850 return ""
851 if output["code"] == "error":
852 return ""
853 clientPath = ""
854 if "path" in output:
855 clientPath = decode_path(output['path'])
856 elif "data" in output:
857 data = output.get("data")
858 lastSpace = data.rfind(b" ")
859 clientPath = decode_path(data[lastSpace + 1:])
861 if clientPath.endswith("..."):
862 clientPath = clientPath[:-3]
863 return clientPath
865 def currentGitBranch():
866 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
868 def isValidGitDir(path):
869 return git_dir(path) != None
871 def parseRevision(ref):
872 return read_pipe("git rev-parse %s" % ref).strip()
874 def branchExists(ref):
875 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
876 ignore_error=True)
877 return len(rev) > 0
879 def extractLogMessageFromGitCommit(commit):
880 logMessage = ""
882 ## fixme: title is first line of commit, not 1st paragraph.
883 foundTitle = False
884 for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
885 if not foundTitle:
886 if len(log) == 1:
887 foundTitle = True
888 continue
890 logMessage += log
891 return logMessage
893 def extractSettingsGitLog(log):
894 values = {}
895 for line in log.split("\n"):
896 line = line.strip()
897 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
898 if not m:
899 continue
901 assignments = m.group(1).split (':')
902 for a in assignments:
903 vals = a.split ('=')
904 key = vals[0].strip()
905 val = ('='.join (vals[1:])).strip()
906 if val.endswith ('\"') and val.startswith('"'):
907 val = val[1:-1]
909 values[key] = val
911 paths = values.get("depot-paths")
912 if not paths:
913 paths = values.get("depot-path")
914 if paths:
915 values['depot-paths'] = paths.split(',')
916 return values
918 def gitBranchExists(branch):
919 proc = subprocess.Popen(["git", "rev-parse", branch],
920 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
921 return proc.wait() == 0;
923 def gitUpdateRef(ref, newvalue):
924 subprocess.check_call(["git", "update-ref", ref, newvalue])
926 def gitDeleteRef(ref):
927 subprocess.check_call(["git", "update-ref", "-d", ref])
929 _gitConfig = {}
931 def gitConfig(key, typeSpecifier=None):
932 if key not in _gitConfig:
933 cmd = [ "git", "config" ]
934 if typeSpecifier:
935 cmd += [ typeSpecifier ]
936 cmd += [ key ]
937 s = read_pipe(cmd, ignore_error=True)
938 _gitConfig[key] = s.strip()
939 return _gitConfig[key]
941 def gitConfigBool(key):
942 """Return a bool, using git config --bool. It is True only if the
943 variable is set to true, and False if set to false or not present
944 in the config."""
946 if key not in _gitConfig:
947 _gitConfig[key] = gitConfig(key, '--bool') == "true"
948 return _gitConfig[key]
950 def gitConfigInt(key):
951 if key not in _gitConfig:
952 cmd = [ "git", "config", "--int", key ]
953 s = read_pipe(cmd, ignore_error=True)
954 v = s.strip()
955 try:
956 _gitConfig[key] = int(gitConfig(key, '--int'))
957 except ValueError:
958 _gitConfig[key] = None
959 return _gitConfig[key]
961 def gitConfigList(key):
962 if key not in _gitConfig:
963 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
964 _gitConfig[key] = s.strip().splitlines()
965 if _gitConfig[key] == ['']:
966 _gitConfig[key] = []
967 return _gitConfig[key]
969 def p4BranchesInGit(branchesAreInRemotes=True):
970 """Find all the branches whose names start with "p4/", looking
971 in remotes or heads as specified by the argument. Return
972 a dictionary of { branch: revision } for each one found.
973 The branch names are the short names, without any
974 "p4/" prefix."""
976 branches = {}
978 cmdline = "git rev-parse --symbolic "
979 if branchesAreInRemotes:
980 cmdline += "--remotes"
981 else:
982 cmdline += "--branches"
984 for line in read_pipe_lines(cmdline):
985 line = line.strip()
987 # only import to p4/
988 if not line.startswith('p4/'):
989 continue
990 # special symbolic ref to p4/master
991 if line == "p4/HEAD":
992 continue
994 # strip off p4/ prefix
995 branch = line[len("p4/"):]
997 branches[branch] = parseRevision(line)
999 return branches
1001 def branch_exists(branch):
1002 """Make sure that the given ref name really exists."""
1004 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
1005 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1006 out, _ = p.communicate()
1007 out = decode_text_stream(out)
1008 if p.returncode:
1009 return False
1010 # expect exactly one line of output: the branch name
1011 return out.rstrip() == branch
1013 def findUpstreamBranchPoint(head = "HEAD"):
1014 branches = p4BranchesInGit()
1015 # map from depot-path to branch name
1016 branchByDepotPath = {}
1017 for branch in branches.keys():
1018 tip = branches[branch]
1019 log = extractLogMessageFromGitCommit(tip)
1020 settings = extractSettingsGitLog(log)
1021 if "depot-paths" in settings:
1022 paths = ",".join(settings["depot-paths"])
1023 branchByDepotPath[paths] = "remotes/p4/" + branch
1025 settings = None
1026 parent = 0
1027 while parent < 65535:
1028 commit = head + "~%s" % parent
1029 log = extractLogMessageFromGitCommit(commit)
1030 settings = extractSettingsGitLog(log)
1031 if "depot-paths" in settings:
1032 paths = ",".join(settings["depot-paths"])
1033 if paths in branchByDepotPath:
1034 return [branchByDepotPath[paths], settings]
1036 parent = parent + 1
1038 return ["", settings]
1040 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
1041 if not silent:
1042 print("Creating/updating branch(es) in %s based on origin branch(es)"
1043 % localRefPrefix)
1045 originPrefix = "origin/p4/"
1047 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
1048 line = line.strip()
1049 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
1050 continue
1052 headName = line[len(originPrefix):]
1053 remoteHead = localRefPrefix + headName
1054 originHead = line
1056 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
1057 if ('depot-paths' not in original
1058 or 'change' not in original):
1059 continue
1061 update = False
1062 if not gitBranchExists(remoteHead):
1063 if verbose:
1064 print("creating %s" % remoteHead)
1065 update = True
1066 else:
1067 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
1068 if 'change' in settings:
1069 if settings['depot-paths'] == original['depot-paths']:
1070 originP4Change = int(original['change'])
1071 p4Change = int(settings['change'])
1072 if originP4Change > p4Change:
1073 print("%s (%s) is newer than %s (%s). "
1074 "Updating p4 branch from origin."
1075 % (originHead, originP4Change,
1076 remoteHead, p4Change))
1077 update = True
1078 else:
1079 print("Ignoring: %s was imported from %s while "
1080 "%s was imported from %s"
1081 % (originHead, ','.join(original['depot-paths']),
1082 remoteHead, ','.join(settings['depot-paths'])))
1084 if update:
1085 system("git update-ref %s %s" % (remoteHead, originHead))
1087 def originP4BranchesExist():
1088 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1091 def p4ParseNumericChangeRange(parts):
1092 changeStart = int(parts[0][1:])
1093 if parts[1] == '#head':
1094 changeEnd = p4_last_change()
1095 else:
1096 changeEnd = int(parts[1])
1098 return (changeStart, changeEnd)
1100 def chooseBlockSize(blockSize):
1101 if blockSize:
1102 return blockSize
1103 else:
1104 return defaultBlockSize
1106 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
1107 assert depotPaths
1109 # Parse the change range into start and end. Try to find integer
1110 # revision ranges as these can be broken up into blocks to avoid
1111 # hitting server-side limits (maxrows, maxscanresults). But if
1112 # that doesn't work, fall back to using the raw revision specifier
1113 # strings, without using block mode.
1115 if changeRange is None or changeRange == '':
1116 changeStart = 1
1117 changeEnd = p4_last_change()
1118 block_size = chooseBlockSize(requestedBlockSize)
1119 else:
1120 parts = changeRange.split(',')
1121 assert len(parts) == 2
1122 try:
1123 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
1124 block_size = chooseBlockSize(requestedBlockSize)
1125 except ValueError:
1126 changeStart = parts[0][1:]
1127 changeEnd = parts[1]
1128 if requestedBlockSize:
1129 die("cannot use --changes-block-size with non-numeric revisions")
1130 block_size = None
1132 changes = set()
1134 # Retrieve changes a block at a time, to prevent running
1135 # into a MaxResults/MaxScanRows error from the server. If
1136 # we _do_ hit one of those errors, turn down the block size
1138 while True:
1139 cmd = ['changes']
1141 if block_size:
1142 end = min(changeEnd, changeStart + block_size)
1143 revisionRange = "%d,%d" % (changeStart, end)
1144 else:
1145 revisionRange = "%s,%s" % (changeStart, changeEnd)
1147 for p in depotPaths:
1148 cmd += ["%s...@%s" % (p, revisionRange)]
1150 # fetch the changes
1151 try:
1152 result = p4CmdList(cmd, errors_as_exceptions=True)
1153 except P4RequestSizeException as e:
1154 if not block_size:
1155 block_size = e.limit
1156 elif block_size > e.limit:
1157 block_size = e.limit
1158 else:
1159 block_size = max(2, block_size // 2)
1161 if verbose: print("block size error, retrying with block size {0}".format(block_size))
1162 continue
1163 except P4Exception as e:
1164 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1166 # Insert changes in chronological order
1167 for entry in reversed(result):
1168 if 'change' not in entry:
1169 continue
1170 changes.add(int(entry['change']))
1172 if not block_size:
1173 break
1175 if end >= changeEnd:
1176 break
1178 changeStart = end + 1
1180 changes = sorted(changes)
1181 return changes
1183 def p4PathStartsWith(path, prefix):
1184 # This method tries to remedy a potential mixed-case issue:
1186 # If UserA adds //depot/DirA/file1
1187 # and UserB adds //depot/dira/file2
1189 # we may or may not have a problem. If you have core.ignorecase=true,
1190 # we treat DirA and dira as the same directory
1191 if gitConfigBool("core.ignorecase"):
1192 return path.lower().startswith(prefix.lower())
1193 return path.startswith(prefix)
1195 def getClientSpec():
1196 """Look at the p4 client spec, create a View() object that contains
1197 all the mappings, and return it."""
1199 specList = p4CmdList("client -o")
1200 if len(specList) != 1:
1201 die('Output from "client -o" is %d lines, expecting 1' %
1202 len(specList))
1204 # dictionary of all client parameters
1205 entry = specList[0]
1207 # the //client/ name
1208 client_name = entry["Client"]
1210 # just the keys that start with "View"
1211 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1213 # hold this new View
1214 view = View(client_name)
1216 # append the lines, in order, to the view
1217 for view_num in range(len(view_keys)):
1218 k = "View%d" % view_num
1219 if k not in view_keys:
1220 die("Expected view key %s missing" % k)
1221 view.append(entry[k])
1223 return view
1225 def getClientRoot():
1226 """Grab the client directory."""
1228 output = p4CmdList("client -o")
1229 if len(output) != 1:
1230 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1232 entry = output[0]
1233 if "Root" not in entry:
1234 die('Client has no "Root"')
1236 return entry["Root"]
1239 # P4 wildcards are not allowed in filenames. P4 complains
1240 # if you simply add them, but you can force it with "-f", in
1241 # which case it translates them into %xx encoding internally.
1243 def wildcard_decode(path):
1244 # Search for and fix just these four characters. Do % last so
1245 # that fixing it does not inadvertently create new %-escapes.
1246 # Cannot have * in a filename in windows; untested as to
1247 # what p4 would do in such a case.
1248 if not platform.system() == "Windows":
1249 path = path.replace("%2A", "*")
1250 path = path.replace("%23", "#") \
1251 .replace("%40", "@") \
1252 .replace("%25", "%")
1253 return path
1255 def wildcard_encode(path):
1256 # do % first to avoid double-encoding the %s introduced here
1257 path = path.replace("%", "%25") \
1258 .replace("*", "%2A") \
1259 .replace("#", "%23") \
1260 .replace("@", "%40")
1261 return path
1263 def wildcard_present(path):
1264 m = re.search("[*#@%]", path)
1265 return m is not None
1267 class LargeFileSystem(object):
1268 """Base class for large file system support."""
1270 def __init__(self, writeToGitStream):
1271 self.largeFiles = set()
1272 self.writeToGitStream = writeToGitStream
1274 def generatePointer(self, cloneDestination, contentFile):
1275 """Return the content of a pointer file that is stored in Git instead of
1276 the actual content."""
1277 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1279 def pushFile(self, localLargeFile):
1280 """Push the actual content which is not stored in the Git repository to
1281 a server."""
1282 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1284 def hasLargeFileExtension(self, relPath):
1285 return functools.reduce(
1286 lambda a, b: a or b,
1287 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1288 False
1291 def generateTempFile(self, contents):
1292 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1293 for d in contents:
1294 contentFile.write(d)
1295 contentFile.close()
1296 return contentFile.name
1298 def exceedsLargeFileThreshold(self, relPath, contents):
1299 if gitConfigInt('git-p4.largeFileThreshold'):
1300 contentsSize = sum(len(d) for d in contents)
1301 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1302 return True
1303 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1304 contentsSize = sum(len(d) for d in contents)
1305 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1306 return False
1307 contentTempFile = self.generateTempFile(contents)
1308 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1309 with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1310 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1311 compressedContentsSize = zf.infolist()[0].compress_size
1312 os.remove(contentTempFile)
1313 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1314 return True
1315 return False
1317 def addLargeFile(self, relPath):
1318 self.largeFiles.add(relPath)
1320 def removeLargeFile(self, relPath):
1321 self.largeFiles.remove(relPath)
1323 def isLargeFile(self, relPath):
1324 return relPath in self.largeFiles
1326 def processContent(self, git_mode, relPath, contents):
1327 """Processes the content of git fast import. This method decides if a
1328 file is stored in the large file system and handles all necessary
1329 steps."""
1330 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1331 contentTempFile = self.generateTempFile(contents)
1332 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1333 if pointer_git_mode:
1334 git_mode = pointer_git_mode
1335 if localLargeFile:
1336 # Move temp file to final location in large file system
1337 largeFileDir = os.path.dirname(localLargeFile)
1338 if not os.path.isdir(largeFileDir):
1339 os.makedirs(largeFileDir)
1340 shutil.move(contentTempFile, localLargeFile)
1341 self.addLargeFile(relPath)
1342 if gitConfigBool('git-p4.largeFilePush'):
1343 self.pushFile(localLargeFile)
1344 if verbose:
1345 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1346 return (git_mode, contents)
1348 class MockLFS(LargeFileSystem):
1349 """Mock large file system for testing."""
1351 def generatePointer(self, contentFile):
1352 """The pointer content is the original content prefixed with "pointer-".
1353 The local filename of the large file storage is derived from the file content.
1355 with open(contentFile, 'r') as f:
1356 content = next(f)
1357 gitMode = '100644'
1358 pointerContents = 'pointer-' + content
1359 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1360 return (gitMode, pointerContents, localLargeFile)
1362 def pushFile(self, localLargeFile):
1363 """The remote filename of the large file storage is the same as the local
1364 one but in a different directory.
1366 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1367 if not os.path.exists(remotePath):
1368 os.makedirs(remotePath)
1369 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1371 class GitLFS(LargeFileSystem):
1372 """Git LFS as backend for the git-p4 large file system.
1373 See https://git-lfs.github.com/ for details."""
1375 def __init__(self, *args):
1376 LargeFileSystem.__init__(self, *args)
1377 self.baseGitAttributes = []
1379 def generatePointer(self, contentFile):
1380 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1381 mode and content which is stored in the Git repository instead of
1382 the actual content. Return also the new location of the actual
1383 content.
1385 if os.path.getsize(contentFile) == 0:
1386 return (None, '', None)
1388 pointerProcess = subprocess.Popen(
1389 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1390 stdout=subprocess.PIPE
1392 pointerFile = decode_text_stream(pointerProcess.stdout.read())
1393 if pointerProcess.wait():
1394 os.remove(contentFile)
1395 die('git-lfs pointer command failed. Did you install the extension?')
1397 # Git LFS removed the preamble in the output of the 'pointer' command
1398 # starting from version 1.2.0. Check for the preamble here to support
1399 # earlier versions.
1400 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1401 if pointerFile.startswith('Git LFS pointer for'):
1402 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1404 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1405 # if someone use external lfs.storage ( not in local repo git )
1406 lfs_path = gitConfig('lfs.storage')
1407 if not lfs_path:
1408 lfs_path = 'lfs'
1409 if not os.path.isabs(lfs_path):
1410 lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
1411 localLargeFile = os.path.join(
1412 lfs_path,
1413 'objects', oid[:2], oid[2:4],
1414 oid,
1416 # LFS Spec states that pointer files should not have the executable bit set.
1417 gitMode = '100644'
1418 return (gitMode, pointerFile, localLargeFile)
1420 def pushFile(self, localLargeFile):
1421 uploadProcess = subprocess.Popen(
1422 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1424 if uploadProcess.wait():
1425 die('git-lfs push command failed. Did you define a remote?')
1427 def generateGitAttributes(self):
1428 return (
1429 self.baseGitAttributes +
1431 '\n',
1432 '#\n',
1433 '# Git LFS (see https://git-lfs.github.com/)\n',
1434 '#\n',
1436 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1437 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1439 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1440 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1444 def addLargeFile(self, relPath):
1445 LargeFileSystem.addLargeFile(self, relPath)
1446 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1448 def removeLargeFile(self, relPath):
1449 LargeFileSystem.removeLargeFile(self, relPath)
1450 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1452 def processContent(self, git_mode, relPath, contents):
1453 if relPath == '.gitattributes':
1454 self.baseGitAttributes = contents
1455 return (git_mode, self.generateGitAttributes())
1456 else:
1457 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1459 class Command:
1460 delete_actions = ( "delete", "move/delete", "purge" )
1461 add_actions = ( "add", "branch", "move/add" )
1463 def __init__(self):
1464 self.usage = "usage: %prog [options]"
1465 self.needsGit = True
1466 self.verbose = False
1468 # This is required for the "append" update_shelve action
1469 def ensure_value(self, attr, value):
1470 if not hasattr(self, attr) or getattr(self, attr) is None:
1471 setattr(self, attr, value)
1472 return getattr(self, attr)
1474 class P4UserMap:
1475 def __init__(self):
1476 self.userMapFromPerforceServer = False
1477 self.myP4UserId = None
1479 def p4UserId(self):
1480 if self.myP4UserId:
1481 return self.myP4UserId
1483 results = p4CmdList("user -o")
1484 for r in results:
1485 if 'User' in r:
1486 self.myP4UserId = r['User']
1487 return r['User']
1488 die("Could not find your p4 user id")
1490 def p4UserIsMe(self, p4User):
1491 # return True if the given p4 user is actually me
1492 me = self.p4UserId()
1493 if not p4User or p4User != me:
1494 return False
1495 else:
1496 return True
1498 def getUserCacheFilename(self):
1499 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1500 return home + "/.gitp4-usercache.txt"
1502 def getUserMapFromPerforceServer(self):
1503 if self.userMapFromPerforceServer:
1504 return
1505 self.users = {}
1506 self.emails = {}
1508 for output in p4CmdList("users"):
1509 if "User" not in output:
1510 continue
1511 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1512 self.emails[output["Email"]] = output["User"]
1514 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1515 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1516 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1517 if mapUser and len(mapUser[0]) == 3:
1518 user = mapUser[0][0]
1519 fullname = mapUser[0][1]
1520 email = mapUser[0][2]
1521 self.users[user] = fullname + " <" + email + ">"
1522 self.emails[email] = user
1524 s = ''
1525 for (key, val) in self.users.items():
1526 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1528 open(self.getUserCacheFilename(), 'w').write(s)
1529 self.userMapFromPerforceServer = True
1531 def loadUserMapFromCache(self):
1532 self.users = {}
1533 self.userMapFromPerforceServer = False
1534 try:
1535 cache = open(self.getUserCacheFilename(), 'r')
1536 lines = cache.readlines()
1537 cache.close()
1538 for line in lines:
1539 entry = line.strip().split("\t")
1540 self.users[entry[0]] = entry[1]
1541 except IOError:
1542 self.getUserMapFromPerforceServer()
1544 class P4Submit(Command, P4UserMap):
1546 conflict_behavior_choices = ("ask", "skip", "quit")
1548 def __init__(self):
1549 Command.__init__(self)
1550 P4UserMap.__init__(self)
1551 self.options = [
1552 optparse.make_option("--origin", dest="origin"),
1553 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1554 # preserve the user, requires relevant p4 permissions
1555 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1556 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1557 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1558 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1559 optparse.make_option("--conflict", dest="conflict_behavior",
1560 choices=self.conflict_behavior_choices),
1561 optparse.make_option("--branch", dest="branch"),
1562 optparse.make_option("--shelve", dest="shelve", action="store_true",
1563 help="Shelve instead of submit. Shelved files are reverted, "
1564 "restoring the workspace to the state before the shelve"),
1565 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1566 metavar="CHANGELIST",
1567 help="update an existing shelved changelist, implies --shelve, "
1568 "repeat in-order for multiple shelved changelists"),
1569 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1570 help="submit only the specified commit(s), one commit or xxx..xxx"),
1571 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1572 help="Disable rebase after submit is completed. Can be useful if you "
1573 "work from a local git branch that is not master"),
1574 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1575 help="Skip Perforce sync of p4/master after submit or shelve"),
1576 optparse.make_option("--no-verify", dest="no_verify", action="store_true",
1577 help="Bypass p4-pre-submit and p4-changelist hooks"),
1579 self.description = """Submit changes from git to the perforce depot.\n
1580 The `p4-pre-submit` hook is executed if it exists and is executable. It
1581 can be bypassed with the `--no-verify` command line option. The hook takes
1582 no parameters and nothing from standard input. Exiting with a non-zero status
1583 from this script prevents `git-p4 submit` from launching.
1585 One usage scenario is to run unit tests in the hook.
1587 The `p4-prepare-changelist` hook is executed right after preparing the default
1588 changelist message and before the editor is started. It takes one parameter,
1589 the name of the file that contains the changelist text. Exiting with a non-zero
1590 status from the script will abort the process.
1592 The purpose of the hook is to edit the message file in place, and it is not
1593 supressed by the `--no-verify` option. This hook is called even if
1594 `--prepare-p4-only` is set.
1596 The `p4-changelist` hook is executed after the changelist message has been
1597 edited by the user. It can be bypassed with the `--no-verify` option. It
1598 takes a single parameter, the name of the file that holds the proposed
1599 changelist text. Exiting with a non-zero status causes the command to abort.
1601 The hook is allowed to edit the changelist file and can be used to normalize
1602 the text into some project standard format. It can also be used to refuse the
1603 Submit after inspect the message file.
1605 The `p4-post-changelist` hook is invoked after the submit has successfully
1606 occurred in P4. It takes no parameters and is meant primarily for notification
1607 and cannot affect the outcome of the git p4 submit action.
1610 self.usage += " [name of git branch to submit into perforce depot]"
1611 self.origin = ""
1612 self.detectRenames = False
1613 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1614 self.dry_run = False
1615 self.shelve = False
1616 self.update_shelve = list()
1617 self.commit = ""
1618 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1619 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1620 self.prepare_p4_only = False
1621 self.conflict_behavior = None
1622 self.isWindows = (platform.system() == "Windows")
1623 self.exportLabels = False
1624 self.p4HasMoveCommand = p4_has_move_command()
1625 self.branch = None
1626 self.no_verify = False
1628 if gitConfig('git-p4.largeFileSystem'):
1629 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1631 def check(self):
1632 if len(p4CmdList("opened ...")) > 0:
1633 die("You have files opened with perforce! Close them before starting the sync.")
1635 def separate_jobs_from_description(self, message):
1636 """Extract and return a possible Jobs field in the commit
1637 message. It goes into a separate section in the p4 change
1638 specification.
1640 A jobs line starts with "Jobs:" and looks like a new field
1641 in a form. Values are white-space separated on the same
1642 line or on following lines that start with a tab.
1644 This does not parse and extract the full git commit message
1645 like a p4 form. It just sees the Jobs: line as a marker
1646 to pass everything from then on directly into the p4 form,
1647 but outside the description section.
1649 Return a tuple (stripped log message, jobs string)."""
1651 m = re.search(r'^Jobs:', message, re.MULTILINE)
1652 if m is None:
1653 return (message, None)
1655 jobtext = message[m.start():]
1656 stripped_message = message[:m.start()].rstrip()
1657 return (stripped_message, jobtext)
1659 def prepareLogMessage(self, template, message, jobs):
1660 """Edits the template returned from "p4 change -o" to insert
1661 the message in the Description field, and the jobs text in
1662 the Jobs field."""
1663 result = ""
1665 inDescriptionSection = False
1667 for line in template.split("\n"):
1668 if line.startswith("#"):
1669 result += line + "\n"
1670 continue
1672 if inDescriptionSection:
1673 if line.startswith("Files:") or line.startswith("Jobs:"):
1674 inDescriptionSection = False
1675 # insert Jobs section
1676 if jobs:
1677 result += jobs + "\n"
1678 else:
1679 continue
1680 else:
1681 if line.startswith("Description:"):
1682 inDescriptionSection = True
1683 line += "\n"
1684 for messageLine in message.split("\n"):
1685 line += "\t" + messageLine + "\n"
1687 result += line + "\n"
1689 return result
1691 def patchRCSKeywords(self, file, regexp):
1692 # Attempt to zap the RCS keywords in a p4 controlled file matching the given regex
1693 (handle, outFileName) = tempfile.mkstemp(dir='.')
1694 try:
1695 with os.fdopen(handle, "wb") as outFile, open(file, "rb") as inFile:
1696 for line in inFile.readlines():
1697 outFile.write(regexp.sub(br'$\1$', line))
1698 # Forcibly overwrite the original file
1699 os.unlink(file)
1700 shutil.move(outFileName, file)
1701 except:
1702 # cleanup our temporary file
1703 os.unlink(outFileName)
1704 print("Failed to strip RCS keywords in %s" % file)
1705 raise
1707 print("Patched up RCS keywords in %s" % file)
1709 def p4UserForCommit(self,id):
1710 # Return the tuple (perforce user,git email) for a given git commit id
1711 self.getUserMapFromPerforceServer()
1712 gitEmail = read_pipe(["git", "log", "--max-count=1",
1713 "--format=%ae", id])
1714 gitEmail = gitEmail.strip()
1715 if gitEmail not in self.emails:
1716 return (None,gitEmail)
1717 else:
1718 return (self.emails[gitEmail],gitEmail)
1720 def checkValidP4Users(self,commits):
1721 # check if any git authors cannot be mapped to p4 users
1722 for id in commits:
1723 (user,email) = self.p4UserForCommit(id)
1724 if not user:
1725 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1726 if gitConfigBool("git-p4.allowMissingP4Users"):
1727 print("%s" % msg)
1728 else:
1729 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1731 def lastP4Changelist(self):
1732 # Get back the last changelist number submitted in this client spec. This
1733 # then gets used to patch up the username in the change. If the same
1734 # client spec is being used by multiple processes then this might go
1735 # wrong.
1736 results = p4CmdList("client -o") # find the current client
1737 client = None
1738 for r in results:
1739 if 'Client' in r:
1740 client = r['Client']
1741 break
1742 if not client:
1743 die("could not get client spec")
1744 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1745 for r in results:
1746 if 'change' in r:
1747 return r['change']
1748 die("Could not get changelist number for last submit - cannot patch up user details")
1750 def modifyChangelistUser(self, changelist, newUser):
1751 # fixup the user field of a changelist after it has been submitted.
1752 changes = p4CmdList("change -o %s" % changelist)
1753 if len(changes) != 1:
1754 die("Bad output from p4 change modifying %s to user %s" %
1755 (changelist, newUser))
1757 c = changes[0]
1758 if c['User'] == newUser: return # nothing to do
1759 c['User'] = newUser
1760 # p4 does not understand format version 3 and above
1761 input = marshal.dumps(c, 2)
1763 result = p4CmdList("change -f -i", stdin=input)
1764 for r in result:
1765 if 'code' in r:
1766 if r['code'] == 'error':
1767 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1768 if 'data' in r:
1769 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1770 return
1771 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1773 def canChangeChangelists(self):
1774 # check to see if we have p4 admin or super-user permissions, either of
1775 # which are required to modify changelists.
1776 results = p4CmdList(["protects", self.depotPath])
1777 for r in results:
1778 if 'perm' in r:
1779 if r['perm'] == 'admin':
1780 return 1
1781 if r['perm'] == 'super':
1782 return 1
1783 return 0
1785 def prepareSubmitTemplate(self, changelist=None):
1786 """Run "p4 change -o" to grab a change specification template.
1787 This does not use "p4 -G", as it is nice to keep the submission
1788 template in original order, since a human might edit it.
1790 Remove lines in the Files section that show changes to files
1791 outside the depot path we're committing into."""
1793 [upstream, settings] = findUpstreamBranchPoint()
1795 template = """\
1796 # A Perforce Change Specification.
1798 # Change: The change number. 'new' on a new changelist.
1799 # Date: The date this specification was last modified.
1800 # Client: The client on which the changelist was created. Read-only.
1801 # User: The user who created the changelist.
1802 # Status: Either 'pending' or 'submitted'. Read-only.
1803 # Type: Either 'public' or 'restricted'. Default is 'public'.
1804 # Description: Comments about the changelist. Required.
1805 # Jobs: What opened jobs are to be closed by this changelist.
1806 # You may delete jobs from this list. (New changelists only.)
1807 # Files: What opened files from the default changelist are to be added
1808 # to this changelist. You may delete files from this list.
1809 # (New changelists only.)
1811 files_list = []
1812 inFilesSection = False
1813 change_entry = None
1814 args = ['change', '-o']
1815 if changelist:
1816 args.append(str(changelist))
1817 for entry in p4CmdList(args):
1818 if 'code' not in entry:
1819 continue
1820 if entry['code'] == 'stat':
1821 change_entry = entry
1822 break
1823 if not change_entry:
1824 die('Failed to decode output of p4 change -o')
1825 for key, value in change_entry.items():
1826 if key.startswith('File'):
1827 if 'depot-paths' in settings:
1828 if not [p for p in settings['depot-paths']
1829 if p4PathStartsWith(value, p)]:
1830 continue
1831 else:
1832 if not p4PathStartsWith(value, self.depotPath):
1833 continue
1834 files_list.append(value)
1835 continue
1836 # Output in the order expected by prepareLogMessage
1837 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1838 if key not in change_entry:
1839 continue
1840 template += '\n'
1841 template += key + ':'
1842 if key == 'Description':
1843 template += '\n'
1844 for field_line in change_entry[key].splitlines():
1845 template += '\t'+field_line+'\n'
1846 if len(files_list) > 0:
1847 template += '\n'
1848 template += 'Files:\n'
1849 for path in files_list:
1850 template += '\t'+path+'\n'
1851 return template
1853 def edit_template(self, template_file):
1854 """Invoke the editor to let the user change the submission
1855 message. Return true if okay to continue with the submit."""
1857 # if configured to skip the editing part, just submit
1858 if gitConfigBool("git-p4.skipSubmitEdit"):
1859 return True
1861 # look at the modification time, to check later if the user saved
1862 # the file
1863 mtime = os.stat(template_file).st_mtime
1865 # invoke the editor
1866 if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1867 editor = os.environ.get("P4EDITOR")
1868 else:
1869 editor = read_pipe("git var GIT_EDITOR").strip()
1870 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1872 # If the file was not saved, prompt to see if this patch should
1873 # be skipped. But skip this verification step if configured so.
1874 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1875 return True
1877 # modification time updated means user saved the file
1878 if os.stat(template_file).st_mtime > mtime:
1879 return True
1881 response = prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1882 if response == 'y':
1883 return True
1884 if response == 'n':
1885 return False
1887 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1888 # diff
1889 if "P4DIFF" in os.environ:
1890 del(os.environ["P4DIFF"])
1891 diff = ""
1892 for editedFile in editedFiles:
1893 diff += p4_read_pipe(['diff', '-du',
1894 wildcard_encode(editedFile)])
1896 # new file diff
1897 newdiff = ""
1898 for newFile in filesToAdd:
1899 newdiff += "==== new file ====\n"
1900 newdiff += "--- /dev/null\n"
1901 newdiff += "+++ %s\n" % newFile
1903 is_link = os.path.islink(newFile)
1904 expect_link = newFile in symlinks
1906 if is_link and expect_link:
1907 newdiff += "+%s\n" % os.readlink(newFile)
1908 else:
1909 f = open(newFile, "r")
1910 try:
1911 for line in f.readlines():
1912 newdiff += "+" + line
1913 except UnicodeDecodeError:
1914 pass # Found non-text data and skip, since diff description should only include text
1915 f.close()
1917 return (diff + newdiff).replace('\r\n', '\n')
1919 def applyCommit(self, id):
1920 """Apply one commit, return True if it succeeded."""
1922 print("Applying", read_pipe(["git", "show", "-s",
1923 "--format=format:%h %s", id]))
1925 (p4User, gitEmail) = self.p4UserForCommit(id)
1927 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1928 filesToAdd = set()
1929 filesToChangeType = set()
1930 filesToDelete = set()
1931 editedFiles = set()
1932 pureRenameCopy = set()
1933 symlinks = set()
1934 filesToChangeExecBit = {}
1935 all_files = list()
1937 for line in diff:
1938 diff = parseDiffTreeEntry(line)
1939 modifier = diff['status']
1940 path = diff['src']
1941 all_files.append(path)
1943 if modifier == "M":
1944 p4_edit(path)
1945 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1946 filesToChangeExecBit[path] = diff['dst_mode']
1947 editedFiles.add(path)
1948 elif modifier == "A":
1949 filesToAdd.add(path)
1950 filesToChangeExecBit[path] = diff['dst_mode']
1951 if path in filesToDelete:
1952 filesToDelete.remove(path)
1954 dst_mode = int(diff['dst_mode'], 8)
1955 if dst_mode == 0o120000:
1956 symlinks.add(path)
1958 elif modifier == "D":
1959 filesToDelete.add(path)
1960 if path in filesToAdd:
1961 filesToAdd.remove(path)
1962 elif modifier == "C":
1963 src, dest = diff['src'], diff['dst']
1964 all_files.append(dest)
1965 p4_integrate(src, dest)
1966 pureRenameCopy.add(dest)
1967 if diff['src_sha1'] != diff['dst_sha1']:
1968 p4_edit(dest)
1969 pureRenameCopy.discard(dest)
1970 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1971 p4_edit(dest)
1972 pureRenameCopy.discard(dest)
1973 filesToChangeExecBit[dest] = diff['dst_mode']
1974 if self.isWindows:
1975 # turn off read-only attribute
1976 os.chmod(dest, stat.S_IWRITE)
1977 os.unlink(dest)
1978 editedFiles.add(dest)
1979 elif modifier == "R":
1980 src, dest = diff['src'], diff['dst']
1981 all_files.append(dest)
1982 if self.p4HasMoveCommand:
1983 p4_edit(src) # src must be open before move
1984 p4_move(src, dest) # opens for (move/delete, move/add)
1985 else:
1986 p4_integrate(src, dest)
1987 if diff['src_sha1'] != diff['dst_sha1']:
1988 p4_edit(dest)
1989 else:
1990 pureRenameCopy.add(dest)
1991 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1992 if not self.p4HasMoveCommand:
1993 p4_edit(dest) # with move: already open, writable
1994 filesToChangeExecBit[dest] = diff['dst_mode']
1995 if not self.p4HasMoveCommand:
1996 if self.isWindows:
1997 os.chmod(dest, stat.S_IWRITE)
1998 os.unlink(dest)
1999 filesToDelete.add(src)
2000 editedFiles.add(dest)
2001 elif modifier == "T":
2002 filesToChangeType.add(path)
2003 else:
2004 die("unknown modifier %s for %s" % (modifier, path))
2006 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
2007 patchcmd = diffcmd + " | git apply "
2008 tryPatchCmd = patchcmd + "--check -"
2009 applyPatchCmd = patchcmd + "--check --apply -"
2010 patch_succeeded = True
2012 if verbose:
2013 print("TryPatch: %s" % tryPatchCmd)
2015 if os.system(tryPatchCmd) != 0:
2016 fixed_rcs_keywords = False
2017 patch_succeeded = False
2018 print("Unfortunately applying the change failed!")
2020 # Patch failed, maybe it's just RCS keyword woes. Look through
2021 # the patch to see if that's possible.
2022 if gitConfigBool("git-p4.attemptRCSCleanup"):
2023 file = None
2024 kwfiles = {}
2025 for file in editedFiles | filesToDelete:
2026 # did this file's delta contain RCS keywords?
2027 regexp = p4_keywords_regexp_for_file(file)
2028 if regexp:
2029 # this file is a possibility...look for RCS keywords.
2030 for line in read_pipe_lines(
2031 ["git", "diff", "%s^..%s" % (id, id), file],
2032 raw=True):
2033 if regexp.search(line):
2034 if verbose:
2035 print("got keyword match on %s in %s in %s" % (regex.pattern, line, file))
2036 kwfiles[file] = regexp
2037 break
2039 for file, regexp in kwfiles.items():
2040 if verbose:
2041 print("zapping %s with %s" % (line, regexp.pattern))
2042 # File is being deleted, so not open in p4. Must
2043 # disable the read-only bit on windows.
2044 if self.isWindows and file not in editedFiles:
2045 os.chmod(file, stat.S_IWRITE)
2046 self.patchRCSKeywords(file, kwfiles[file])
2047 fixed_rcs_keywords = True
2049 if fixed_rcs_keywords:
2050 print("Retrying the patch with RCS keywords cleaned up")
2051 if os.system(tryPatchCmd) == 0:
2052 patch_succeeded = True
2053 print("Patch succeesed this time with RCS keywords cleaned")
2055 if not patch_succeeded:
2056 for f in editedFiles:
2057 p4_revert(f)
2058 return False
2061 # Apply the patch for real, and do add/delete/+x handling.
2063 system(applyPatchCmd)
2065 for f in filesToChangeType:
2066 p4_edit(f, "-t", "auto")
2067 for f in filesToAdd:
2068 p4_add(f)
2069 for f in filesToDelete:
2070 p4_revert(f)
2071 p4_delete(f)
2073 # Set/clear executable bits
2074 for f in filesToChangeExecBit.keys():
2075 mode = filesToChangeExecBit[f]
2076 setP4ExecBit(f, mode)
2078 update_shelve = 0
2079 if len(self.update_shelve) > 0:
2080 update_shelve = self.update_shelve.pop(0)
2081 p4_reopen_in_change(update_shelve, all_files)
2084 # Build p4 change description, starting with the contents
2085 # of the git commit message.
2087 logMessage = extractLogMessageFromGitCommit(id)
2088 logMessage = logMessage.strip()
2089 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
2091 template = self.prepareSubmitTemplate(update_shelve)
2092 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
2094 if self.preserveUser:
2095 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
2097 if self.checkAuthorship and not self.p4UserIsMe(p4User):
2098 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2099 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2100 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2102 separatorLine = "######## everything below this line is just the diff #######\n"
2103 if not self.prepare_p4_only:
2104 submitTemplate += separatorLine
2105 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2107 (handle, fileName) = tempfile.mkstemp()
2108 tmpFile = os.fdopen(handle, "w+b")
2109 if self.isWindows:
2110 submitTemplate = submitTemplate.replace("\n", "\r\n")
2111 tmpFile.write(encode_text_stream(submitTemplate))
2112 tmpFile.close()
2114 submitted = False
2116 try:
2117 # Allow the hook to edit the changelist text before presenting it
2118 # to the user.
2119 if not run_git_hook("p4-prepare-changelist", [fileName]):
2120 return False
2122 if self.prepare_p4_only:
2124 # Leave the p4 tree prepared, and the submit template around
2125 # and let the user decide what to do next
2127 submitted = True
2128 print("")
2129 print("P4 workspace prepared for submission.")
2130 print("To submit or revert, go to client workspace")
2131 print(" " + self.clientPath)
2132 print("")
2133 print("To submit, use \"p4 submit\" to write a new description,")
2134 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2135 " \"git p4\"." % fileName)
2136 print("You can delete the file \"%s\" when finished." % fileName)
2138 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2139 print("To preserve change ownership by user %s, you must\n" \
2140 "do \"p4 change -f <change>\" after submitting and\n" \
2141 "edit the User field.")
2142 if pureRenameCopy:
2143 print("After submitting, renamed files must be re-synced.")
2144 print("Invoke \"p4 sync -f\" on each of these files:")
2145 for f in pureRenameCopy:
2146 print(" " + f)
2148 print("")
2149 print("To revert the changes, use \"p4 revert ...\", and delete")
2150 print("the submit template file \"%s\"" % fileName)
2151 if filesToAdd:
2152 print("Since the commit adds new files, they must be deleted:")
2153 for f in filesToAdd:
2154 print(" " + f)
2155 print("")
2156 sys.stdout.flush()
2157 return True
2159 if self.edit_template(fileName):
2160 if not self.no_verify:
2161 if not run_git_hook("p4-changelist", [fileName]):
2162 print("The p4-changelist hook failed.")
2163 sys.stdout.flush()
2164 return False
2166 # read the edited message and submit
2167 tmpFile = open(fileName, "rb")
2168 message = decode_text_stream(tmpFile.read())
2169 tmpFile.close()
2170 if self.isWindows:
2171 message = message.replace("\r\n", "\n")
2172 if message.find(separatorLine) != -1:
2173 submitTemplate = message[:message.index(separatorLine)]
2174 else:
2175 submitTemplate = message
2177 if len(submitTemplate.strip()) == 0:
2178 print("Changelist is empty, aborting this changelist.")
2179 sys.stdout.flush()
2180 return False
2182 if update_shelve:
2183 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2184 elif self.shelve:
2185 p4_write_pipe(['shelve', '-i'], submitTemplate)
2186 else:
2187 p4_write_pipe(['submit', '-i'], submitTemplate)
2188 # The rename/copy happened by applying a patch that created a
2189 # new file. This leaves it writable, which confuses p4.
2190 for f in pureRenameCopy:
2191 p4_sync(f, "-f")
2193 if self.preserveUser:
2194 if p4User:
2195 # Get last changelist number. Cannot easily get it from
2196 # the submit command output as the output is
2197 # unmarshalled.
2198 changelist = self.lastP4Changelist()
2199 self.modifyChangelistUser(changelist, p4User)
2201 submitted = True
2203 run_git_hook("p4-post-changelist")
2204 finally:
2205 # Revert changes if we skip this patch
2206 if not submitted or self.shelve:
2207 if self.shelve:
2208 print ("Reverting shelved files.")
2209 else:
2210 print ("Submission cancelled, undoing p4 changes.")
2211 sys.stdout.flush()
2212 for f in editedFiles | filesToDelete:
2213 p4_revert(f)
2214 for f in filesToAdd:
2215 p4_revert(f)
2216 os.remove(f)
2218 if not self.prepare_p4_only:
2219 os.remove(fileName)
2220 return submitted
2222 # Export git tags as p4 labels. Create a p4 label and then tag
2223 # with that.
2224 def exportGitTags(self, gitTags):
2225 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2226 if len(validLabelRegexp) == 0:
2227 validLabelRegexp = defaultLabelRegexp
2228 m = re.compile(validLabelRegexp)
2230 for name in gitTags:
2232 if not m.match(name):
2233 if verbose:
2234 print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2235 continue
2237 # Get the p4 commit this corresponds to
2238 logMessage = extractLogMessageFromGitCommit(name)
2239 values = extractSettingsGitLog(logMessage)
2241 if 'change' not in values:
2242 # a tag pointing to something not sent to p4; ignore
2243 if verbose:
2244 print("git tag %s does not give a p4 commit" % name)
2245 continue
2246 else:
2247 changelist = values['change']
2249 # Get the tag details.
2250 inHeader = True
2251 isAnnotated = False
2252 body = []
2253 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2254 l = l.strip()
2255 if inHeader:
2256 if re.match(r'tag\s+', l):
2257 isAnnotated = True
2258 elif re.match(r'\s*$', l):
2259 inHeader = False
2260 continue
2261 else:
2262 body.append(l)
2264 if not isAnnotated:
2265 body = ["lightweight tag imported by git p4\n"]
2267 # Create the label - use the same view as the client spec we are using
2268 clientSpec = getClientSpec()
2270 labelTemplate = "Label: %s\n" % name
2271 labelTemplate += "Description:\n"
2272 for b in body:
2273 labelTemplate += "\t" + b + "\n"
2274 labelTemplate += "View:\n"
2275 for depot_side in clientSpec.mappings:
2276 labelTemplate += "\t%s\n" % depot_side
2278 if self.dry_run:
2279 print("Would create p4 label %s for tag" % name)
2280 elif self.prepare_p4_only:
2281 print("Not creating p4 label %s for tag due to option" \
2282 " --prepare-p4-only" % name)
2283 else:
2284 p4_write_pipe(["label", "-i"], labelTemplate)
2286 # Use the label
2287 p4_system(["tag", "-l", name] +
2288 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2290 if verbose:
2291 print("created p4 label for tag %s" % name)
2293 def run(self, args):
2294 if len(args) == 0:
2295 self.master = currentGitBranch()
2296 elif len(args) == 1:
2297 self.master = args[0]
2298 if not branchExists(self.master):
2299 die("Branch %s does not exist" % self.master)
2300 else:
2301 return False
2303 for i in self.update_shelve:
2304 if i <= 0:
2305 sys.exit("invalid changelist %d" % i)
2307 if self.master:
2308 allowSubmit = gitConfig("git-p4.allowSubmit")
2309 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2310 die("%s is not in git-p4.allowSubmit" % self.master)
2312 [upstream, settings] = findUpstreamBranchPoint()
2313 self.depotPath = settings['depot-paths'][0]
2314 if len(self.origin) == 0:
2315 self.origin = upstream
2317 if len(self.update_shelve) > 0:
2318 self.shelve = True
2320 if self.preserveUser:
2321 if not self.canChangeChangelists():
2322 die("Cannot preserve user names without p4 super-user or admin permissions")
2324 # if not set from the command line, try the config file
2325 if self.conflict_behavior is None:
2326 val = gitConfig("git-p4.conflict")
2327 if val:
2328 if val not in self.conflict_behavior_choices:
2329 die("Invalid value '%s' for config git-p4.conflict" % val)
2330 else:
2331 val = "ask"
2332 self.conflict_behavior = val
2334 if self.verbose:
2335 print("Origin branch is " + self.origin)
2337 if len(self.depotPath) == 0:
2338 print("Internal error: cannot locate perforce depot path from existing branches")
2339 sys.exit(128)
2341 self.useClientSpec = False
2342 if gitConfigBool("git-p4.useclientspec"):
2343 self.useClientSpec = True
2344 if self.useClientSpec:
2345 self.clientSpecDirs = getClientSpec()
2347 # Check for the existence of P4 branches
2348 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2350 if self.useClientSpec and not branchesDetected:
2351 # all files are relative to the client spec
2352 self.clientPath = getClientRoot()
2353 else:
2354 self.clientPath = p4Where(self.depotPath)
2356 if self.clientPath == "":
2357 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2359 print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2360 self.oldWorkingDirectory = os.getcwd()
2362 # ensure the clientPath exists
2363 new_client_dir = False
2364 if not os.path.exists(self.clientPath):
2365 new_client_dir = True
2366 os.makedirs(self.clientPath)
2368 chdir(self.clientPath, is_client_path=True)
2369 if self.dry_run:
2370 print("Would synchronize p4 checkout in %s" % self.clientPath)
2371 else:
2372 print("Synchronizing p4 checkout...")
2373 if new_client_dir:
2374 # old one was destroyed, and maybe nobody told p4
2375 p4_sync("...", "-f")
2376 else:
2377 p4_sync("...")
2378 self.check()
2380 commits = []
2381 if self.master:
2382 committish = self.master
2383 else:
2384 committish = 'HEAD'
2386 if self.commit != "":
2387 if self.commit.find("..") != -1:
2388 limits_ish = self.commit.split("..")
2389 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2390 commits.append(line.strip())
2391 commits.reverse()
2392 else:
2393 commits.append(self.commit)
2394 else:
2395 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2396 commits.append(line.strip())
2397 commits.reverse()
2399 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2400 self.checkAuthorship = False
2401 else:
2402 self.checkAuthorship = True
2404 if self.preserveUser:
2405 self.checkValidP4Users(commits)
2408 # Build up a set of options to be passed to diff when
2409 # submitting each commit to p4.
2411 if self.detectRenames:
2412 # command-line -M arg
2413 self.diffOpts = "-M"
2414 else:
2415 # If not explicitly set check the config variable
2416 detectRenames = gitConfig("git-p4.detectRenames")
2418 if detectRenames.lower() == "false" or detectRenames == "":
2419 self.diffOpts = ""
2420 elif detectRenames.lower() == "true":
2421 self.diffOpts = "-M"
2422 else:
2423 self.diffOpts = "-M%s" % detectRenames
2425 # no command-line arg for -C or --find-copies-harder, just
2426 # config variables
2427 detectCopies = gitConfig("git-p4.detectCopies")
2428 if detectCopies.lower() == "false" or detectCopies == "":
2429 pass
2430 elif detectCopies.lower() == "true":
2431 self.diffOpts += " -C"
2432 else:
2433 self.diffOpts += " -C%s" % detectCopies
2435 if gitConfigBool("git-p4.detectCopiesHarder"):
2436 self.diffOpts += " --find-copies-harder"
2438 num_shelves = len(self.update_shelve)
2439 if num_shelves > 0 and num_shelves != len(commits):
2440 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2441 (len(commits), num_shelves))
2443 if not self.no_verify:
2444 try:
2445 if not run_git_hook("p4-pre-submit"):
2446 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " \
2447 "this pre-submission check by adding\nthe command line option '--no-verify', " \
2448 "however,\nthis will also skip the p4-changelist hook as well.")
2449 sys.exit(1)
2450 except Exception as e:
2451 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "\
2452 "with the error '{0}'".format(e.message) )
2453 sys.exit(1)
2456 # Apply the commits, one at a time. On failure, ask if should
2457 # continue to try the rest of the patches, or quit.
2459 if self.dry_run:
2460 print("Would apply")
2461 applied = []
2462 last = len(commits) - 1
2463 for i, commit in enumerate(commits):
2464 if self.dry_run:
2465 print(" ", read_pipe(["git", "show", "-s",
2466 "--format=format:%h %s", commit]))
2467 ok = True
2468 else:
2469 ok = self.applyCommit(commit)
2470 if ok:
2471 applied.append(commit)
2472 if self.prepare_p4_only:
2473 if i < last:
2474 print("Processing only the first commit due to option" \
2475 " --prepare-p4-only")
2476 break
2477 else:
2478 if i < last:
2479 # prompt for what to do, or use the option/variable
2480 if self.conflict_behavior == "ask":
2481 print("What do you want to do?")
2482 response = prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2483 elif self.conflict_behavior == "skip":
2484 response = "s"
2485 elif self.conflict_behavior == "quit":
2486 response = "q"
2487 else:
2488 die("Unknown conflict_behavior '%s'" %
2489 self.conflict_behavior)
2491 if response == "s":
2492 print("Skipping this commit, but applying the rest")
2493 if response == "q":
2494 print("Quitting")
2495 break
2497 chdir(self.oldWorkingDirectory)
2498 shelved_applied = "shelved" if self.shelve else "applied"
2499 if self.dry_run:
2500 pass
2501 elif self.prepare_p4_only:
2502 pass
2503 elif len(commits) == len(applied):
2504 print("All commits {0}!".format(shelved_applied))
2506 sync = P4Sync()
2507 if self.branch:
2508 sync.branch = self.branch
2509 if self.disable_p4sync:
2510 sync.sync_origin_only()
2511 else:
2512 sync.run([])
2514 if not self.disable_rebase:
2515 rebase = P4Rebase()
2516 rebase.rebase()
2518 else:
2519 if len(applied) == 0:
2520 print("No commits {0}.".format(shelved_applied))
2521 else:
2522 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2523 for c in commits:
2524 if c in applied:
2525 star = "*"
2526 else:
2527 star = " "
2528 print(star, read_pipe(["git", "show", "-s",
2529 "--format=format:%h %s", c]))
2530 print("You will have to do 'git p4 sync' and rebase.")
2532 if gitConfigBool("git-p4.exportLabels"):
2533 self.exportLabels = True
2535 if self.exportLabels:
2536 p4Labels = getP4Labels(self.depotPath)
2537 gitTags = getGitTags()
2539 missingGitTags = gitTags - p4Labels
2540 self.exportGitTags(missingGitTags)
2542 # exit with error unless everything applied perfectly
2543 if len(commits) != len(applied):
2544 sys.exit(1)
2546 return True
2548 class View(object):
2549 """Represent a p4 view ("p4 help views"), and map files in a
2550 repo according to the view."""
2552 def __init__(self, client_name):
2553 self.mappings = []
2554 self.client_prefix = "//%s/" % client_name
2555 # cache results of "p4 where" to lookup client file locations
2556 self.client_spec_path_cache = {}
2558 def append(self, view_line):
2559 """Parse a view line, splitting it into depot and client
2560 sides. Append to self.mappings, preserving order. This
2561 is only needed for tag creation."""
2563 # Split the view line into exactly two words. P4 enforces
2564 # structure on these lines that simplifies this quite a bit.
2566 # Either or both words may be double-quoted.
2567 # Single quotes do not matter.
2568 # Double-quote marks cannot occur inside the words.
2569 # A + or - prefix is also inside the quotes.
2570 # There are no quotes unless they contain a space.
2571 # The line is already white-space stripped.
2572 # The two words are separated by a single space.
2574 if view_line[0] == '"':
2575 # First word is double quoted. Find its end.
2576 close_quote_index = view_line.find('"', 1)
2577 if close_quote_index <= 0:
2578 die("No first-word closing quote found: %s" % view_line)
2579 depot_side = view_line[1:close_quote_index]
2580 # skip closing quote and space
2581 rhs_index = close_quote_index + 1 + 1
2582 else:
2583 space_index = view_line.find(" ")
2584 if space_index <= 0:
2585 die("No word-splitting space found: %s" % view_line)
2586 depot_side = view_line[0:space_index]
2587 rhs_index = space_index + 1
2589 # prefix + means overlay on previous mapping
2590 if depot_side.startswith("+"):
2591 depot_side = depot_side[1:]
2593 # prefix - means exclude this path, leave out of mappings
2594 exclude = False
2595 if depot_side.startswith("-"):
2596 exclude = True
2597 depot_side = depot_side[1:]
2599 if not exclude:
2600 self.mappings.append(depot_side)
2602 def convert_client_path(self, clientFile):
2603 # chop off //client/ part to make it relative
2604 if not decode_path(clientFile).startswith(self.client_prefix):
2605 die("No prefix '%s' on clientFile '%s'" %
2606 (self.client_prefix, clientFile))
2607 return clientFile[len(self.client_prefix):]
2609 def update_client_spec_path_cache(self, files):
2610 """ Caching file paths by "p4 where" batch query """
2612 # List depot file paths exclude that already cached
2613 fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]
2615 if len(fileArgs) == 0:
2616 return # All files in cache
2618 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2619 for res in where_result:
2620 if "code" in res and res["code"] == "error":
2621 # assume error is "... file(s) not in client view"
2622 continue
2623 if "clientFile" not in res:
2624 die("No clientFile in 'p4 where' output")
2625 if "unmap" in res:
2626 # it will list all of them, but only one not unmap-ped
2627 continue
2628 depot_path = decode_path(res['depotFile'])
2629 if gitConfigBool("core.ignorecase"):
2630 depot_path = depot_path.lower()
2631 self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])
2633 # not found files or unmap files set to ""
2634 for depotFile in fileArgs:
2635 depotFile = decode_path(depotFile)
2636 if gitConfigBool("core.ignorecase"):
2637 depotFile = depotFile.lower()
2638 if depotFile not in self.client_spec_path_cache:
2639 self.client_spec_path_cache[depotFile] = b''
2641 def map_in_client(self, depot_path):
2642 """Return the relative location in the client where this
2643 depot file should live. Returns "" if the file should
2644 not be mapped in the client."""
2646 if gitConfigBool("core.ignorecase"):
2647 depot_path = depot_path.lower()
2649 if depot_path in self.client_spec_path_cache:
2650 return self.client_spec_path_cache[depot_path]
2652 die( "Error: %s is not found in client spec path" % depot_path )
2653 return ""
2655 def cloneExcludeCallback(option, opt_str, value, parser):
2656 # prepend "/" because the first "/" was consumed as part of the option itself.
2657 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2658 parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2660 class P4Sync(Command, P4UserMap):
2662 def __init__(self):
2663 Command.__init__(self)
2664 P4UserMap.__init__(self)
2665 self.options = [
2666 optparse.make_option("--branch", dest="branch"),
2667 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2668 optparse.make_option("--changesfile", dest="changesFile"),
2669 optparse.make_option("--silent", dest="silent", action="store_true"),
2670 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2671 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2672 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2673 help="Import into refs/heads/ , not refs/remotes"),
2674 optparse.make_option("--max-changes", dest="maxChanges",
2675 help="Maximum number of changes to import"),
2676 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2677 help="Internal block size to use when iteratively calling p4 changes"),
2678 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2679 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2680 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2681 help="Only sync files that are included in the Perforce Client Spec"),
2682 optparse.make_option("-/", dest="cloneExclude",
2683 action="callback", callback=cloneExcludeCallback, type="string",
2684 help="exclude depot path"),
2686 self.description = """Imports from Perforce into a git repository.\n
2687 example:
2688 //depot/my/project/ -- to import the current head
2689 //depot/my/project/@all -- to import everything
2690 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2692 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2694 self.usage += " //depot/path[@revRange]"
2695 self.silent = False
2696 self.createdBranches = set()
2697 self.committedChanges = set()
2698 self.branch = ""
2699 self.detectBranches = False
2700 self.detectLabels = False
2701 self.importLabels = False
2702 self.changesFile = ""
2703 self.syncWithOrigin = True
2704 self.importIntoRemotes = True
2705 self.maxChanges = ""
2706 self.changes_block_size = None
2707 self.keepRepoPath = False
2708 self.depotPaths = None
2709 self.p4BranchesInGit = []
2710 self.cloneExclude = []
2711 self.useClientSpec = False
2712 self.useClientSpec_from_options = False
2713 self.clientSpecDirs = None
2714 self.tempBranches = []
2715 self.tempBranchLocation = "refs/git-p4-tmp"
2716 self.largeFileSystem = None
2717 self.suppress_meta_comment = False
2719 if gitConfig('git-p4.largeFileSystem'):
2720 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2721 self.largeFileSystem = largeFileSystemConstructor(
2722 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2725 if gitConfig("git-p4.syncFromOrigin") == "false":
2726 self.syncWithOrigin = False
2728 self.depotPaths = []
2729 self.changeRange = ""
2730 self.previousDepotPaths = []
2731 self.hasOrigin = False
2733 # map from branch depot path to parent branch
2734 self.knownBranches = {}
2735 self.initialParents = {}
2737 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2738 self.labels = {}
2740 # Force a checkpoint in fast-import and wait for it to finish
2741 def checkpoint(self):
2742 self.gitStream.write("checkpoint\n\n")
2743 self.gitStream.write("progress checkpoint\n\n")
2744 self.gitStream.flush()
2745 out = self.gitOutput.readline()
2746 if self.verbose:
2747 print("checkpoint finished: " + out)
2749 def isPathWanted(self, path):
2750 for p in self.cloneExclude:
2751 if p.endswith("/"):
2752 if p4PathStartsWith(path, p):
2753 return False
2754 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2755 elif path.lower() == p.lower():
2756 return False
2757 for p in self.depotPaths:
2758 if p4PathStartsWith(path, decode_path(p)):
2759 return True
2760 return False
2762 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2763 files = []
2764 fnum = 0
2765 while "depotFile%s" % fnum in commit:
2766 path = commit["depotFile%s" % fnum]
2767 found = self.isPathWanted(decode_path(path))
2768 if not found:
2769 fnum = fnum + 1
2770 continue
2772 file = {}
2773 file["path"] = path
2774 file["rev"] = commit["rev%s" % fnum]
2775 file["action"] = commit["action%s" % fnum]
2776 file["type"] = commit["type%s" % fnum]
2777 if shelved:
2778 file["shelved_cl"] = int(shelved_cl)
2779 files.append(file)
2780 fnum = fnum + 1
2781 return files
2783 def extractJobsFromCommit(self, commit):
2784 jobs = []
2785 jnum = 0
2786 while "job%s" % jnum in commit:
2787 job = commit["job%s" % jnum]
2788 jobs.append(job)
2789 jnum = jnum + 1
2790 return jobs
2792 def stripRepoPath(self, path, prefixes):
2793 """When streaming files, this is called to map a p4 depot path
2794 to where it should go in git. The prefixes are either
2795 self.depotPaths, or self.branchPrefixes in the case of
2796 branch detection."""
2798 if self.useClientSpec:
2799 # branch detection moves files up a level (the branch name)
2800 # from what client spec interpretation gives
2801 path = decode_path(self.clientSpecDirs.map_in_client(path))
2802 if self.detectBranches:
2803 for b in self.knownBranches:
2804 if p4PathStartsWith(path, b + "/"):
2805 path = path[len(b)+1:]
2807 elif self.keepRepoPath:
2808 # Preserve everything in relative path name except leading
2809 # //depot/; just look at first prefix as they all should
2810 # be in the same depot.
2811 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2812 if p4PathStartsWith(path, depot):
2813 path = path[len(depot):]
2815 else:
2816 for p in prefixes:
2817 if p4PathStartsWith(path, p):
2818 path = path[len(p):]
2819 break
2821 path = wildcard_decode(path)
2822 return path
2824 def splitFilesIntoBranches(self, commit):
2825 """Look at each depotFile in the commit to figure out to what
2826 branch it belongs."""
2828 if self.clientSpecDirs:
2829 files = self.extractFilesFromCommit(commit)
2830 self.clientSpecDirs.update_client_spec_path_cache(files)
2832 branches = {}
2833 fnum = 0
2834 while "depotFile%s" % fnum in commit:
2835 raw_path = commit["depotFile%s" % fnum]
2836 path = decode_path(raw_path)
2837 found = self.isPathWanted(path)
2838 if not found:
2839 fnum = fnum + 1
2840 continue
2842 file = {}
2843 file["path"] = raw_path
2844 file["rev"] = commit["rev%s" % fnum]
2845 file["action"] = commit["action%s" % fnum]
2846 file["type"] = commit["type%s" % fnum]
2847 fnum = fnum + 1
2849 # start with the full relative path where this file would
2850 # go in a p4 client
2851 if self.useClientSpec:
2852 relPath = decode_path(self.clientSpecDirs.map_in_client(path))
2853 else:
2854 relPath = self.stripRepoPath(path, self.depotPaths)
2856 for branch in self.knownBranches.keys():
2857 # add a trailing slash so that a commit into qt/4.2foo
2858 # doesn't end up in qt/4.2, e.g.
2859 if p4PathStartsWith(relPath, branch + "/"):
2860 if branch not in branches:
2861 branches[branch] = []
2862 branches[branch].append(file)
2863 break
2865 return branches
2867 def writeToGitStream(self, gitMode, relPath, contents):
2868 self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath)))
2869 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2870 for d in contents:
2871 self.gitStream.write(d)
2872 self.gitStream.write('\n')
2874 def encodeWithUTF8(self, path):
2875 try:
2876 path.decode('ascii')
2877 except:
2878 encoding = 'utf8'
2879 if gitConfig('git-p4.pathEncoding'):
2880 encoding = gitConfig('git-p4.pathEncoding')
2881 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2882 if self.verbose:
2883 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2884 return path
2886 # output one file from the P4 stream
2887 # - helper for streamP4Files
2889 def streamOneP4File(self, file, contents):
2890 file_path = file['depotFile']
2891 relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)
2893 if verbose:
2894 if 'fileSize' in self.stream_file:
2895 size = int(self.stream_file['fileSize'])
2896 else:
2897 size = 0 # deleted files don't get a fileSize apparently
2898 sys.stdout.write('\r%s --> %s (%s)\n' % (
2899 file_path, relPath, format_size_human_readable(size)))
2900 sys.stdout.flush()
2902 (type_base, type_mods) = split_p4_type(file["type"])
2904 git_mode = "100644"
2905 if "x" in type_mods:
2906 git_mode = "100755"
2907 if type_base == "symlink":
2908 git_mode = "120000"
2909 # p4 print on a symlink sometimes contains "target\n";
2910 # if it does, remove the newline
2911 data = ''.join(decode_text_stream(c) for c in contents)
2912 if not data:
2913 # Some version of p4 allowed creating a symlink that pointed
2914 # to nothing. This causes p4 errors when checking out such
2915 # a change, and errors here too. Work around it by ignoring
2916 # the bad symlink; hopefully a future change fixes it.
2917 print("\nIgnoring empty symlink in %s" % file_path)
2918 return
2919 elif data[-1] == '\n':
2920 contents = [data[:-1]]
2921 else:
2922 contents = [data]
2924 if type_base == "utf16":
2925 # p4 delivers different text in the python output to -G
2926 # than it does when using "print -o", or normal p4 client
2927 # operations. utf16 is converted to ascii or utf8, perhaps.
2928 # But ascii text saved as -t utf16 is completely mangled.
2929 # Invoke print -o to get the real contents.
2931 # On windows, the newlines will always be mangled by print, so put
2932 # them back too. This is not needed to the cygwin windows version,
2933 # just the native "NT" type.
2935 try:
2936 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)
2937 except Exception as e:
2938 if 'Translation of file content failed' in str(e):
2939 type_base = 'binary'
2940 else:
2941 raise e
2942 else:
2943 if p4_version_string().find('/NT') >= 0:
2944 text = text.replace(b'\r\n', b'\n')
2945 contents = [ text ]
2947 if type_base == "apple":
2948 # Apple filetype files will be streamed as a concatenation of
2949 # its appledouble header and the contents. This is useless
2950 # on both macs and non-macs. If using "print -q -o xx", it
2951 # will create "xx" with the data, and "%xx" with the header.
2952 # This is also not very useful.
2954 # Ideally, someday, this script can learn how to generate
2955 # appledouble files directly and import those to git, but
2956 # non-mac machines can never find a use for apple filetype.
2957 print("\nIgnoring apple filetype file %s" % file['depotFile'])
2958 return
2960 # Note that we do not try to de-mangle keywords on utf16 files,
2961 # even though in theory somebody may want that.
2962 regexp = p4_keywords_regexp_for_type(type_base, type_mods)
2963 if regexp:
2964 contents = [regexp.sub(br'$\1$', c) for c in contents]
2966 if self.largeFileSystem:
2967 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2969 self.writeToGitStream(git_mode, relPath, contents)
2971 def streamOneP4Deletion(self, file):
2972 relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)
2973 if verbose:
2974 sys.stdout.write("delete %s\n" % relPath)
2975 sys.stdout.flush()
2976 self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath)))
2978 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2979 self.largeFileSystem.removeLargeFile(relPath)
2981 # handle another chunk of streaming data
2982 def streamP4FilesCb(self, marshalled):
2984 # catch p4 errors and complain
2985 err = None
2986 if "code" in marshalled:
2987 if marshalled["code"] == "error":
2988 if "data" in marshalled:
2989 err = marshalled["data"].rstrip()
2991 if not err and 'fileSize' in self.stream_file:
2992 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2993 if required_bytes > 0:
2994 err = 'Not enough space left on %s! Free at least %s.' % (
2995 os.getcwd(), format_size_human_readable(required_bytes))
2997 if err:
2998 f = None
2999 if self.stream_have_file_info:
3000 if "depotFile" in self.stream_file:
3001 f = self.stream_file["depotFile"]
3002 # force a failure in fast-import, else an empty
3003 # commit will be made
3004 self.gitStream.write("\n")
3005 self.gitStream.write("die-now\n")
3006 self.gitStream.close()
3007 # ignore errors, but make sure it exits first
3008 self.importProcess.wait()
3009 if f:
3010 die("Error from p4 print for %s: %s" % (f, err))
3011 else:
3012 die("Error from p4 print: %s" % err)
3014 if 'depotFile' in marshalled and self.stream_have_file_info:
3015 # start of a new file - output the old one first
3016 self.streamOneP4File(self.stream_file, self.stream_contents)
3017 self.stream_file = {}
3018 self.stream_contents = []
3019 self.stream_have_file_info = False
3021 # pick up the new file information... for the
3022 # 'data' field we need to append to our array
3023 for k in marshalled.keys():
3024 if k == 'data':
3025 if 'streamContentSize' not in self.stream_file:
3026 self.stream_file['streamContentSize'] = 0
3027 self.stream_file['streamContentSize'] += len(marshalled['data'])
3028 self.stream_contents.append(marshalled['data'])
3029 else:
3030 self.stream_file[k] = marshalled[k]
3032 if (verbose and
3033 'streamContentSize' in self.stream_file and
3034 'fileSize' in self.stream_file and
3035 'depotFile' in self.stream_file):
3036 size = int(self.stream_file["fileSize"])
3037 if size > 0:
3038 progress = 100*self.stream_file['streamContentSize']/size
3039 sys.stdout.write('\r%s %d%% (%s)' % (
3040 self.stream_file['depotFile'], progress,
3041 format_size_human_readable(size)))
3042 sys.stdout.flush()
3044 self.stream_have_file_info = True
3046 # Stream directly from "p4 files" into "git fast-import"
3047 def streamP4Files(self, files):
3048 filesForCommit = []
3049 filesToRead = []
3050 filesToDelete = []
3052 for f in files:
3053 filesForCommit.append(f)
3054 if f['action'] in self.delete_actions:
3055 filesToDelete.append(f)
3056 else:
3057 filesToRead.append(f)
3059 # deleted files...
3060 for f in filesToDelete:
3061 self.streamOneP4Deletion(f)
3063 if len(filesToRead) > 0:
3064 self.stream_file = {}
3065 self.stream_contents = []
3066 self.stream_have_file_info = False
3068 # curry self argument
3069 def streamP4FilesCbSelf(entry):
3070 self.streamP4FilesCb(entry)
3072 fileArgs = []
3073 for f in filesToRead:
3074 if 'shelved_cl' in f:
3075 # Handle shelved CLs using the "p4 print file@=N" syntax to print
3076 # the contents
3077 fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl']))
3078 else:
3079 fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev']))
3081 fileArgs.append(fileArg)
3083 p4CmdList(["-x", "-", "print"],
3084 stdin=fileArgs,
3085 cb=streamP4FilesCbSelf)
3087 # do the last chunk
3088 if 'depotFile' in self.stream_file:
3089 self.streamOneP4File(self.stream_file, self.stream_contents)
3091 def make_email(self, userid):
3092 if userid in self.users:
3093 return self.users[userid]
3094 else:
3095 return "%s <a@b>" % userid
3097 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
3098 """ Stream a p4 tag.
3099 commit is either a git commit, or a fast-import mark, ":<p4commit>"
3102 if verbose:
3103 print("writing tag %s for commit %s" % (labelName, commit))
3104 gitStream.write("tag %s\n" % labelName)
3105 gitStream.write("from %s\n" % commit)
3107 if 'Owner' in labelDetails:
3108 owner = labelDetails["Owner"]
3109 else:
3110 owner = None
3112 # Try to use the owner of the p4 label, or failing that,
3113 # the current p4 user id.
3114 if owner:
3115 email = self.make_email(owner)
3116 else:
3117 email = self.make_email(self.p4UserId())
3118 tagger = "%s %s %s" % (email, epoch, self.tz)
3120 gitStream.write("tagger %s\n" % tagger)
3122 print("labelDetails=",labelDetails)
3123 if 'Description' in labelDetails:
3124 description = labelDetails['Description']
3125 else:
3126 description = 'Label from git p4'
3128 gitStream.write("data %d\n" % len(description))
3129 gitStream.write(description)
3130 gitStream.write("\n")
3132 def inClientSpec(self, path):
3133 if not self.clientSpecDirs:
3134 return True
3135 inClientSpec = self.clientSpecDirs.map_in_client(path)
3136 if not inClientSpec and self.verbose:
3137 print('Ignoring file outside of client spec: {0}'.format(path))
3138 return inClientSpec
3140 def hasBranchPrefix(self, path):
3141 if not self.branchPrefixes:
3142 return True
3143 hasPrefix = [p for p in self.branchPrefixes
3144 if p4PathStartsWith(path, p)]
3145 if not hasPrefix and self.verbose:
3146 print('Ignoring file outside of prefix: {0}'.format(path))
3147 return hasPrefix
3149 def findShadowedFiles(self, files, change):
3150 # Perforce allows you commit files and directories with the same name,
3151 # so you could have files //depot/foo and //depot/foo/bar both checked
3152 # in. A p4 sync of a repository in this state fails. Deleting one of
3153 # the files recovers the repository.
3155 # Git will not allow the broken state to exist and only the most recent
3156 # of the conflicting names is left in the repository. When one of the
3157 # conflicting files is deleted we need to re-add the other one to make
3158 # sure the git repository recovers in the same way as perforce.
3159 deleted = [f for f in files if f['action'] in self.delete_actions]
3160 to_check = set()
3161 for f in deleted:
3162 path = decode_path(f['path'])
3163 to_check.add(path + '/...')
3164 while True:
3165 path = path.rsplit("/", 1)[0]
3166 if path == "/" or path in to_check:
3167 break
3168 to_check.add(path)
3169 to_check = ['%s@%s' % (wildcard_encode(p), change) for p in to_check
3170 if self.hasBranchPrefix(p)]
3171 if to_check:
3172 stat_result = p4CmdList(["-x", "-", "fstat", "-T",
3173 "depotFile,headAction,headRev,headType"], stdin=to_check)
3174 for record in stat_result:
3175 if record['code'] != 'stat':
3176 continue
3177 if record['headAction'] in self.delete_actions:
3178 continue
3179 files.append({
3180 'action': 'add',
3181 'path': record['depotFile'],
3182 'rev': record['headRev'],
3183 'type': record['headType']})
3185 def commit(self, details, files, branch, parent = "", allow_empty=False):
3186 epoch = details["time"]
3187 author = details["user"]
3188 jobs = self.extractJobsFromCommit(details)
3190 if self.verbose:
3191 print('commit into {0}'.format(branch))
3193 files = [f for f in files
3194 if self.hasBranchPrefix(decode_path(f['path']))]
3195 self.findShadowedFiles(files, details['change'])
3197 if self.clientSpecDirs:
3198 self.clientSpecDirs.update_client_spec_path_cache(files)
3200 files = [f for f in files if self.inClientSpec(decode_path(f['path']))]
3202 if gitConfigBool('git-p4.keepEmptyCommits'):
3203 allow_empty = True
3205 if not files and not allow_empty:
3206 print('Ignoring revision {0} as it would produce an empty commit.'
3207 .format(details['change']))
3208 return
3210 self.gitStream.write("commit %s\n" % branch)
3211 self.gitStream.write("mark :%s\n" % details["change"])
3212 self.committedChanges.add(int(details["change"]))
3213 committer = ""
3214 if author not in self.users:
3215 self.getUserMapFromPerforceServer()
3216 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3218 self.gitStream.write("committer %s\n" % committer)
3220 self.gitStream.write("data <<EOT\n")
3221 self.gitStream.write(details["desc"])
3222 if len(jobs) > 0:
3223 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3225 if not self.suppress_meta_comment:
3226 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3227 (','.join(self.branchPrefixes), details["change"]))
3228 if len(details['options']) > 0:
3229 self.gitStream.write(": options = %s" % details['options'])
3230 self.gitStream.write("]\n")
3232 self.gitStream.write("EOT\n\n")
3234 if len(parent) > 0:
3235 if self.verbose:
3236 print("parent %s" % parent)
3237 self.gitStream.write("from %s\n" % parent)
3239 self.streamP4Files(files)
3240 self.gitStream.write("\n")
3242 change = int(details["change"])
3244 if change in self.labels:
3245 label = self.labels[change]
3246 labelDetails = label[0]
3247 labelRevisions = label[1]
3248 if self.verbose:
3249 print("Change %s is labelled %s" % (change, labelDetails))
3251 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3252 for p in self.branchPrefixes])
3254 if len(files) == len(labelRevisions):
3256 cleanedFiles = {}
3257 for info in files:
3258 if info["action"] in self.delete_actions:
3259 continue
3260 cleanedFiles[info["depotFile"]] = info["rev"]
3262 if cleanedFiles == labelRevisions:
3263 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3265 else:
3266 if not self.silent:
3267 print("Tag %s does not match with change %s: files do not match."
3268 % (labelDetails["label"], change))
3270 else:
3271 if not self.silent:
3272 print("Tag %s does not match with change %s: file count is different."
3273 % (labelDetails["label"], change))
3275 # Build a dictionary of changelists and labels, for "detect-labels" option.
3276 def getLabels(self):
3277 self.labels = {}
3279 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3280 if len(l) > 0 and not self.silent:
3281 print("Finding files belonging to labels in %s" % self.depotPaths)
3283 for output in l:
3284 label = output["label"]
3285 revisions = {}
3286 newestChange = 0
3287 if self.verbose:
3288 print("Querying files for label %s" % label)
3289 for file in p4CmdList(["files"] +
3290 ["%s...@%s" % (p, label)
3291 for p in self.depotPaths]):
3292 revisions[file["depotFile"]] = file["rev"]
3293 change = int(file["change"])
3294 if change > newestChange:
3295 newestChange = change
3297 self.labels[newestChange] = [output, revisions]
3299 if self.verbose:
3300 print("Label changes: %s" % self.labels.keys())
3302 # Import p4 labels as git tags. A direct mapping does not
3303 # exist, so assume that if all the files are at the same revision
3304 # then we can use that, or it's something more complicated we should
3305 # just ignore.
3306 def importP4Labels(self, stream, p4Labels):
3307 if verbose:
3308 print("import p4 labels: " + ' '.join(p4Labels))
3310 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3311 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3312 if len(validLabelRegexp) == 0:
3313 validLabelRegexp = defaultLabelRegexp
3314 m = re.compile(validLabelRegexp)
3316 for name in p4Labels:
3317 commitFound = False
3319 if not m.match(name):
3320 if verbose:
3321 print("label %s does not match regexp %s" % (name,validLabelRegexp))
3322 continue
3324 if name in ignoredP4Labels:
3325 continue
3327 labelDetails = p4CmdList(['label', "-o", name])[0]
3329 # get the most recent changelist for each file in this label
3330 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3331 for p in self.depotPaths])
3333 if 'change' in change:
3334 # find the corresponding git commit; take the oldest commit
3335 changelist = int(change['change'])
3336 if changelist in self.committedChanges:
3337 gitCommit = ":%d" % changelist # use a fast-import mark
3338 commitFound = True
3339 else:
3340 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3341 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3342 if len(gitCommit) == 0:
3343 print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3344 else:
3345 commitFound = True
3346 gitCommit = gitCommit.strip()
3348 if commitFound:
3349 # Convert from p4 time format
3350 try:
3351 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3352 except ValueError:
3353 print("Could not convert label time %s" % labelDetails['Update'])
3354 tmwhen = 1
3356 when = int(time.mktime(tmwhen))
3357 self.streamTag(stream, name, labelDetails, gitCommit, when)
3358 if verbose:
3359 print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3360 else:
3361 if verbose:
3362 print("Label %s has no changelists - possibly deleted?" % name)
3364 if not commitFound:
3365 # We can't import this label; don't try again as it will get very
3366 # expensive repeatedly fetching all the files for labels that will
3367 # never be imported. If the label is moved in the future, the
3368 # ignore will need to be removed manually.
3369 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3371 def guessProjectName(self):
3372 for p in self.depotPaths:
3373 if p.endswith("/"):
3374 p = p[:-1]
3375 p = p[p.strip().rfind("/") + 1:]
3376 if not p.endswith("/"):
3377 p += "/"
3378 return p
3380 def getBranchMapping(self):
3381 lostAndFoundBranches = set()
3383 user = gitConfig("git-p4.branchUser")
3384 if len(user) > 0:
3385 command = "branches -u %s" % user
3386 else:
3387 command = "branches"
3389 for info in p4CmdList(command):
3390 details = p4Cmd(["branch", "-o", info["branch"]])
3391 viewIdx = 0
3392 while "View%s" % viewIdx in details:
3393 paths = details["View%s" % viewIdx].split(" ")
3394 viewIdx = viewIdx + 1
3395 # require standard //depot/foo/... //depot/bar/... mapping
3396 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3397 continue
3398 source = paths[0]
3399 destination = paths[1]
3400 ## HACK
3401 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3402 source = source[len(self.depotPaths[0]):-4]
3403 destination = destination[len(self.depotPaths[0]):-4]
3405 if destination in self.knownBranches:
3406 if not self.silent:
3407 print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3408 print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3409 continue
3411 self.knownBranches[destination] = source
3413 lostAndFoundBranches.discard(destination)
3415 if source not in self.knownBranches:
3416 lostAndFoundBranches.add(source)
3418 # Perforce does not strictly require branches to be defined, so we also
3419 # check git config for a branch list.
3421 # Example of branch definition in git config file:
3422 # [git-p4]
3423 # branchList=main:branchA
3424 # branchList=main:branchB
3425 # branchList=branchA:branchC
3426 configBranches = gitConfigList("git-p4.branchList")
3427 for branch in configBranches:
3428 if branch:
3429 (source, destination) = branch.split(":")
3430 self.knownBranches[destination] = source
3432 lostAndFoundBranches.discard(destination)
3434 if source not in self.knownBranches:
3435 lostAndFoundBranches.add(source)
3438 for branch in lostAndFoundBranches:
3439 self.knownBranches[branch] = branch
3441 def getBranchMappingFromGitBranches(self):
3442 branches = p4BranchesInGit(self.importIntoRemotes)
3443 for branch in branches.keys():
3444 if branch == "master":
3445 branch = "main"
3446 else:
3447 branch = branch[len(self.projectName):]
3448 self.knownBranches[branch] = branch
3450 def updateOptionDict(self, d):
3451 option_keys = {}
3452 if self.keepRepoPath:
3453 option_keys['keepRepoPath'] = 1
3455 d["options"] = ' '.join(sorted(option_keys.keys()))
3457 def readOptions(self, d):
3458 self.keepRepoPath = ('options' in d
3459 and ('keepRepoPath' in d['options']))
3461 def gitRefForBranch(self, branch):
3462 if branch == "main":
3463 return self.refPrefix + "master"
3465 if len(branch) <= 0:
3466 return branch
3468 return self.refPrefix + self.projectName + branch
3470 def gitCommitByP4Change(self, ref, change):
3471 if self.verbose:
3472 print("looking in ref " + ref + " for change %s using bisect..." % change)
3474 earliestCommit = ""
3475 latestCommit = parseRevision(ref)
3477 while True:
3478 if self.verbose:
3479 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3480 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3481 if len(next) == 0:
3482 if self.verbose:
3483 print("argh")
3484 return ""
3485 log = extractLogMessageFromGitCommit(next)
3486 settings = extractSettingsGitLog(log)
3487 currentChange = int(settings['change'])
3488 if self.verbose:
3489 print("current change %s" % currentChange)
3491 if currentChange == change:
3492 if self.verbose:
3493 print("found %s" % next)
3494 return next
3496 if currentChange < change:
3497 earliestCommit = "^%s" % next
3498 else:
3499 if next == latestCommit:
3500 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3501 latestCommit = "%s^@" % next
3503 return ""
3505 def importNewBranch(self, branch, maxChange):
3506 # make fast-import flush all changes to disk and update the refs using the checkpoint
3507 # command so that we can try to find the branch parent in the git history
3508 self.gitStream.write("checkpoint\n\n");
3509 self.gitStream.flush();
3510 branchPrefix = self.depotPaths[0] + branch + "/"
3511 range = "@1,%s" % maxChange
3512 #print "prefix" + branchPrefix
3513 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3514 if len(changes) <= 0:
3515 return False
3516 firstChange = changes[0]
3517 #print "first change in branch: %s" % firstChange
3518 sourceBranch = self.knownBranches[branch]
3519 sourceDepotPath = self.depotPaths[0] + sourceBranch
3520 sourceRef = self.gitRefForBranch(sourceBranch)
3521 #print "source " + sourceBranch
3523 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3524 #print "branch parent: %s" % branchParentChange
3525 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3526 if len(gitParent) > 0:
3527 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3528 #print "parent git commit: %s" % gitParent
3530 self.importChanges(changes)
3531 return True
3533 def searchParent(self, parent, branch, target):
3534 targetTree = read_pipe(["git", "rev-parse",
3535 "{}^{{tree}}".format(target)]).strip()
3536 for line in read_pipe_lines(["git", "rev-list", "--format=%H %T",
3537 "--no-merges", parent]):
3538 if line.startswith("commit "):
3539 continue
3540 commit, tree = line.strip().split(" ")
3541 if tree == targetTree:
3542 if self.verbose:
3543 print("Found parent of %s in commit %s" % (branch, commit))
3544 return commit
3545 return None
3547 def importChanges(self, changes, origin_revision=0):
3548 cnt = 1
3549 for change in changes:
3550 description = p4_describe(change)
3551 self.updateOptionDict(description)
3553 if not self.silent:
3554 sys.stdout.write("\rImporting revision %s (%d%%)" % (
3555 change, (cnt * 100) // len(changes)))
3556 sys.stdout.flush()
3557 cnt = cnt + 1
3559 try:
3560 if self.detectBranches:
3561 branches = self.splitFilesIntoBranches(description)
3562 for branch in branches.keys():
3563 ## HACK --hwn
3564 branchPrefix = self.depotPaths[0] + branch + "/"
3565 self.branchPrefixes = [ branchPrefix ]
3567 parent = ""
3569 filesForCommit = branches[branch]
3571 if self.verbose:
3572 print("branch is %s" % branch)
3574 self.updatedBranches.add(branch)
3576 if branch not in self.createdBranches:
3577 self.createdBranches.add(branch)
3578 parent = self.knownBranches[branch]
3579 if parent == branch:
3580 parent = ""
3581 else:
3582 fullBranch = self.projectName + branch
3583 if fullBranch not in self.p4BranchesInGit:
3584 if not self.silent:
3585 print("\n Importing new branch %s" % fullBranch);
3586 if self.importNewBranch(branch, change - 1):
3587 parent = ""
3588 self.p4BranchesInGit.append(fullBranch)
3589 if not self.silent:
3590 print("\n Resuming with change %s" % change);
3592 if self.verbose:
3593 print("parent determined through known branches: %s" % parent)
3595 branch = self.gitRefForBranch(branch)
3596 parent = self.gitRefForBranch(parent)
3598 if self.verbose:
3599 print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3601 if len(parent) == 0 and branch in self.initialParents:
3602 parent = self.initialParents[branch]
3603 del self.initialParents[branch]
3605 blob = None
3606 if len(parent) > 0:
3607 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3608 if self.verbose:
3609 print("Creating temporary branch: " + tempBranch)
3610 self.commit(description, filesForCommit, tempBranch)
3611 self.tempBranches.append(tempBranch)
3612 self.checkpoint()
3613 blob = self.searchParent(parent, branch, tempBranch)
3614 if blob:
3615 self.commit(description, filesForCommit, branch, blob)
3616 else:
3617 if self.verbose:
3618 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3619 self.commit(description, filesForCommit, branch, parent)
3620 else:
3621 files = self.extractFilesFromCommit(description)
3622 self.commit(description, files, self.branch,
3623 self.initialParent)
3624 # only needed once, to connect to the previous commit
3625 self.initialParent = ""
3626 except IOError:
3627 print(self.gitError.read())
3628 sys.exit(1)
3630 def sync_origin_only(self):
3631 if self.syncWithOrigin:
3632 self.hasOrigin = originP4BranchesExist()
3633 if self.hasOrigin:
3634 if not self.silent:
3635 print('Syncing with origin first, using "git fetch origin"')
3636 system("git fetch origin")
3638 def importHeadRevision(self, revision):
3639 print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3641 details = {}
3642 details["user"] = "git perforce import user"
3643 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3644 % (' '.join(self.depotPaths), revision))
3645 details["change"] = revision
3646 newestRevision = 0
3648 fileCnt = 0
3649 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3651 for info in p4CmdList(["files"] + fileArgs):
3653 if 'code' in info and info['code'] == 'error':
3654 sys.stderr.write("p4 returned an error: %s\n"
3655 % info['data'])
3656 if info['data'].find("must refer to client") >= 0:
3657 sys.stderr.write("This particular p4 error is misleading.\n")
3658 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3659 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3660 sys.exit(1)
3661 if 'p4ExitCode' in info:
3662 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3663 sys.exit(1)
3666 change = int(info["change"])
3667 if change > newestRevision:
3668 newestRevision = change
3670 if info["action"] in self.delete_actions:
3671 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3672 #fileCnt = fileCnt + 1
3673 continue
3675 for prop in ["depotFile", "rev", "action", "type" ]:
3676 details["%s%s" % (prop, fileCnt)] = info[prop]
3678 fileCnt = fileCnt + 1
3680 details["change"] = newestRevision
3682 # Use time from top-most change so that all git p4 clones of
3683 # the same p4 repo have the same commit SHA1s.
3684 res = p4_describe(newestRevision)
3685 details["time"] = res["time"]
3687 self.updateOptionDict(details)
3688 try:
3689 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3690 except IOError as err:
3691 print("IO error with git fast-import. Is your git version recent enough?")
3692 print("IO error details: {}".format(err))
3693 print(self.gitError.read())
3696 def importRevisions(self, args, branch_arg_given):
3697 changes = []
3699 if len(self.changesFile) > 0:
3700 with open(self.changesFile) as f:
3701 output = f.readlines()
3702 changeSet = set()
3703 for line in output:
3704 changeSet.add(int(line))
3706 for change in changeSet:
3707 changes.append(change)
3709 changes.sort()
3710 else:
3711 # catch "git p4 sync" with no new branches, in a repo that
3712 # does not have any existing p4 branches
3713 if len(args) == 0:
3714 if not self.p4BranchesInGit:
3715 raise P4CommandException("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3717 # The default branch is master, unless --branch is used to
3718 # specify something else. Make sure it exists, or complain
3719 # nicely about how to use --branch.
3720 if not self.detectBranches:
3721 if not branch_exists(self.branch):
3722 if branch_arg_given:
3723 raise P4CommandException("Error: branch %s does not exist." % self.branch)
3724 else:
3725 raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3726 self.branch)
3728 if self.verbose:
3729 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3730 self.changeRange))
3731 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3733 if len(self.maxChanges) > 0:
3734 changes = changes[:min(int(self.maxChanges), len(changes))]
3736 if len(changes) == 0:
3737 if not self.silent:
3738 print("No changes to import!")
3739 else:
3740 if not self.silent and not self.detectBranches:
3741 print("Import destination: %s" % self.branch)
3743 self.updatedBranches = set()
3745 if not self.detectBranches:
3746 if args:
3747 # start a new branch
3748 self.initialParent = ""
3749 else:
3750 # build on a previous revision
3751 self.initialParent = parseRevision(self.branch)
3753 self.importChanges(changes)
3755 if not self.silent:
3756 print("")
3757 if len(self.updatedBranches) > 0:
3758 sys.stdout.write("Updated branches: ")
3759 for b in self.updatedBranches:
3760 sys.stdout.write("%s " % b)
3761 sys.stdout.write("\n")
3763 def openStreams(self):
3764 self.importProcess = subprocess.Popen(["git", "fast-import"],
3765 stdin=subprocess.PIPE,
3766 stdout=subprocess.PIPE,
3767 stderr=subprocess.PIPE);
3768 self.gitOutput = self.importProcess.stdout
3769 self.gitStream = self.importProcess.stdin
3770 self.gitError = self.importProcess.stderr
3772 if bytes is not str:
3773 # Wrap gitStream.write() so that it can be called using `str` arguments
3774 def make_encoded_write(write):
3775 def encoded_write(s):
3776 return write(s.encode() if isinstance(s, str) else s)
3777 return encoded_write
3779 self.gitStream.write = make_encoded_write(self.gitStream.write)
3781 def closeStreams(self):
3782 if self.gitStream is None:
3783 return
3784 self.gitStream.close()
3785 if self.importProcess.wait() != 0:
3786 die("fast-import failed: %s" % self.gitError.read())
3787 self.gitOutput.close()
3788 self.gitError.close()
3789 self.gitStream = None
3791 def run(self, args):
3792 if self.importIntoRemotes:
3793 self.refPrefix = "refs/remotes/p4/"
3794 else:
3795 self.refPrefix = "refs/heads/p4/"
3797 self.sync_origin_only()
3799 branch_arg_given = bool(self.branch)
3800 if len(self.branch) == 0:
3801 self.branch = self.refPrefix + "master"
3802 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3803 system("git update-ref %s refs/heads/p4" % self.branch)
3804 system("git branch -D p4")
3806 # accept either the command-line option, or the configuration variable
3807 if self.useClientSpec:
3808 # will use this after clone to set the variable
3809 self.useClientSpec_from_options = True
3810 else:
3811 if gitConfigBool("git-p4.useclientspec"):
3812 self.useClientSpec = True
3813 if self.useClientSpec:
3814 self.clientSpecDirs = getClientSpec()
3816 # TODO: should always look at previous commits,
3817 # merge with previous imports, if possible.
3818 if args == []:
3819 if self.hasOrigin:
3820 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3822 # branches holds mapping from branch name to sha1
3823 branches = p4BranchesInGit(self.importIntoRemotes)
3825 # restrict to just this one, disabling detect-branches
3826 if branch_arg_given:
3827 short = self.branch.split("/")[-1]
3828 if short in branches:
3829 self.p4BranchesInGit = [ short ]
3830 else:
3831 self.p4BranchesInGit = branches.keys()
3833 if len(self.p4BranchesInGit) > 1:
3834 if not self.silent:
3835 print("Importing from/into multiple branches")
3836 self.detectBranches = True
3837 for branch in branches.keys():
3838 self.initialParents[self.refPrefix + branch] = \
3839 branches[branch]
3841 if self.verbose:
3842 print("branches: %s" % self.p4BranchesInGit)
3844 p4Change = 0
3845 for branch in self.p4BranchesInGit:
3846 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3848 settings = extractSettingsGitLog(logMsg)
3850 self.readOptions(settings)
3851 if ('depot-paths' in settings
3852 and 'change' in settings):
3853 change = int(settings['change']) + 1
3854 p4Change = max(p4Change, change)
3856 depotPaths = sorted(settings['depot-paths'])
3857 if self.previousDepotPaths == []:
3858 self.previousDepotPaths = depotPaths
3859 else:
3860 paths = []
3861 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3862 prev_list = prev.split("/")
3863 cur_list = cur.split("/")
3864 for i in range(0, min(len(cur_list), len(prev_list))):
3865 if cur_list[i] != prev_list[i]:
3866 i = i - 1
3867 break
3869 paths.append ("/".join(cur_list[:i + 1]))
3871 self.previousDepotPaths = paths
3873 if p4Change > 0:
3874 self.depotPaths = sorted(self.previousDepotPaths)
3875 self.changeRange = "@%s,#head" % p4Change
3876 if not self.silent and not self.detectBranches:
3877 print("Performing incremental import into %s git branch" % self.branch)
3879 # accept multiple ref name abbreviations:
3880 # refs/foo/bar/branch -> use it exactly
3881 # p4/branch -> prepend refs/remotes/ or refs/heads/
3882 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3883 if not self.branch.startswith("refs/"):
3884 if self.importIntoRemotes:
3885 prepend = "refs/remotes/"
3886 else:
3887 prepend = "refs/heads/"
3888 if not self.branch.startswith("p4/"):
3889 prepend += "p4/"
3890 self.branch = prepend + self.branch
3892 if len(args) == 0 and self.depotPaths:
3893 if not self.silent:
3894 print("Depot paths: %s" % ' '.join(self.depotPaths))
3895 else:
3896 if self.depotPaths and self.depotPaths != args:
3897 print("previous import used depot path %s and now %s was specified. "
3898 "This doesn't work!" % (' '.join (self.depotPaths),
3899 ' '.join (args)))
3900 sys.exit(1)
3902 self.depotPaths = sorted(args)
3904 revision = ""
3905 self.users = {}
3907 # Make sure no revision specifiers are used when --changesfile
3908 # is specified.
3909 bad_changesfile = False
3910 if len(self.changesFile) > 0:
3911 for p in self.depotPaths:
3912 if p.find("@") >= 0 or p.find("#") >= 0:
3913 bad_changesfile = True
3914 break
3915 if bad_changesfile:
3916 die("Option --changesfile is incompatible with revision specifiers")
3918 newPaths = []
3919 for p in self.depotPaths:
3920 if p.find("@") != -1:
3921 atIdx = p.index("@")
3922 self.changeRange = p[atIdx:]
3923 if self.changeRange == "@all":
3924 self.changeRange = ""
3925 elif ',' not in self.changeRange:
3926 revision = self.changeRange
3927 self.changeRange = ""
3928 p = p[:atIdx]
3929 elif p.find("#") != -1:
3930 hashIdx = p.index("#")
3931 revision = p[hashIdx:]
3932 p = p[:hashIdx]
3933 elif self.previousDepotPaths == []:
3934 # pay attention to changesfile, if given, else import
3935 # the entire p4 tree at the head revision
3936 if len(self.changesFile) == 0:
3937 revision = "#head"
3939 p = re.sub ("\.\.\.$", "", p)
3940 if not p.endswith("/"):
3941 p += "/"
3943 newPaths.append(p)
3945 self.depotPaths = newPaths
3947 # --detect-branches may change this for each branch
3948 self.branchPrefixes = self.depotPaths
3950 self.loadUserMapFromCache()
3951 self.labels = {}
3952 if self.detectLabels:
3953 self.getLabels();
3955 if self.detectBranches:
3956 ## FIXME - what's a P4 projectName ?
3957 self.projectName = self.guessProjectName()
3959 if self.hasOrigin:
3960 self.getBranchMappingFromGitBranches()
3961 else:
3962 self.getBranchMapping()
3963 if self.verbose:
3964 print("p4-git branches: %s" % self.p4BranchesInGit)
3965 print("initial parents: %s" % self.initialParents)
3966 for b in self.p4BranchesInGit:
3967 if b != "master":
3969 ## FIXME
3970 b = b[len(self.projectName):]
3971 self.createdBranches.add(b)
3973 p4_check_access()
3975 self.openStreams()
3977 err = None
3979 try:
3980 if revision:
3981 self.importHeadRevision(revision)
3982 else:
3983 self.importRevisions(args, branch_arg_given)
3985 if gitConfigBool("git-p4.importLabels"):
3986 self.importLabels = True
3988 if self.importLabels:
3989 p4Labels = getP4Labels(self.depotPaths)
3990 gitTags = getGitTags()
3992 missingP4Labels = p4Labels - gitTags
3993 self.importP4Labels(self.gitStream, missingP4Labels)
3995 except P4CommandException as e:
3996 err = e
3998 finally:
3999 self.closeStreams()
4001 if err:
4002 die(str(err))
4004 # Cleanup temporary branches created during import
4005 if self.tempBranches != []:
4006 for branch in self.tempBranches:
4007 read_pipe("git update-ref -d %s" % branch)
4008 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
4010 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
4011 # a convenient shortcut refname "p4".
4012 if self.importIntoRemotes:
4013 head_ref = self.refPrefix + "HEAD"
4014 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
4015 system(["git", "symbolic-ref", head_ref, self.branch])
4017 return True
4019 class P4Rebase(Command):
4020 def __init__(self):
4021 Command.__init__(self)
4022 self.options = [
4023 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
4025 self.importLabels = False
4026 self.description = ("Fetches the latest revision from perforce and "
4027 + "rebases the current work (branch) against it")
4029 def run(self, args):
4030 sync = P4Sync()
4031 sync.importLabels = self.importLabels
4032 sync.run([])
4034 return self.rebase()
4036 def rebase(self):
4037 if os.system("git update-index --refresh") != 0:
4038 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
4039 if len(read_pipe("git diff-index HEAD --")) > 0:
4040 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
4042 [upstream, settings] = findUpstreamBranchPoint()
4043 if len(upstream) == 0:
4044 die("Cannot find upstream branchpoint for rebase")
4046 # the branchpoint may be p4/foo~3, so strip off the parent
4047 upstream = re.sub("~[0-9]+$", "", upstream)
4049 print("Rebasing the current branch onto %s" % upstream)
4050 oldHead = read_pipe("git rev-parse HEAD").strip()
4051 system("git rebase %s" % upstream)
4052 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
4053 return True
4055 class P4Clone(P4Sync):
4056 def __init__(self):
4057 P4Sync.__init__(self)
4058 self.description = "Creates a new git repository and imports from Perforce into it"
4059 self.usage = "usage: %prog [options] //depot/path[@revRange]"
4060 self.options += [
4061 optparse.make_option("--destination", dest="cloneDestination",
4062 action='store', default=None,
4063 help="where to leave result of the clone"),
4064 optparse.make_option("--bare", dest="cloneBare",
4065 action="store_true", default=False),
4067 self.cloneDestination = None
4068 self.needsGit = False
4069 self.cloneBare = False
4071 def defaultDestination(self, args):
4072 ## TODO: use common prefix of args?
4073 depotPath = args[0]
4074 depotDir = re.sub("(@[^@]*)$", "", depotPath)
4075 depotDir = re.sub("(#[^#]*)$", "", depotDir)
4076 depotDir = re.sub(r"\.\.\.$", "", depotDir)
4077 depotDir = re.sub(r"/$", "", depotDir)
4078 return os.path.split(depotDir)[1]
4080 def run(self, args):
4081 if len(args) < 1:
4082 return False
4084 if self.keepRepoPath and not self.cloneDestination:
4085 sys.stderr.write("Must specify destination for --keep-path\n")
4086 sys.exit(1)
4088 depotPaths = args
4090 if not self.cloneDestination and len(depotPaths) > 1:
4091 self.cloneDestination = depotPaths[-1]
4092 depotPaths = depotPaths[:-1]
4094 for p in depotPaths:
4095 if not p.startswith("//"):
4096 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
4097 return False
4099 if not self.cloneDestination:
4100 self.cloneDestination = self.defaultDestination(args)
4102 print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
4104 if not os.path.exists(self.cloneDestination):
4105 os.makedirs(self.cloneDestination)
4106 chdir(self.cloneDestination)
4108 init_cmd = [ "git", "init" ]
4109 if self.cloneBare:
4110 init_cmd.append("--bare")
4111 retcode = subprocess.call(init_cmd)
4112 if retcode:
4113 raise CalledProcessError(retcode, init_cmd)
4115 if not P4Sync.run(self, depotPaths):
4116 return False
4118 # create a master branch and check out a work tree
4119 if gitBranchExists(self.branch):
4120 system([ "git", "branch", currentGitBranch(), self.branch ])
4121 if not self.cloneBare:
4122 system([ "git", "checkout", "-f" ])
4123 else:
4124 print('Not checking out any branch, use ' \
4125 '"git checkout -q -b master <branch>"')
4127 # auto-set this variable if invoked with --use-client-spec
4128 if self.useClientSpec_from_options:
4129 system("git config --bool git-p4.useclientspec true")
4131 return True
4133 class P4Unshelve(Command):
4134 def __init__(self):
4135 Command.__init__(self)
4136 self.options = []
4137 self.origin = "HEAD"
4138 self.description = "Unshelve a P4 changelist into a git commit"
4139 self.usage = "usage: %prog [options] changelist"
4140 self.options += [
4141 optparse.make_option("--origin", dest="origin",
4142 help="Use this base revision instead of the default (%s)" % self.origin),
4144 self.verbose = False
4145 self.noCommit = False
4146 self.destbranch = "refs/remotes/p4-unshelved"
4148 def renameBranch(self, branch_name):
4149 """ Rename the existing branch to branch_name.N
4152 found = True
4153 for i in range(0,1000):
4154 backup_branch_name = "{0}.{1}".format(branch_name, i)
4155 if not gitBranchExists(backup_branch_name):
4156 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
4157 gitDeleteRef(branch_name)
4158 found = True
4159 print("renamed old unshelve branch to {0}".format(backup_branch_name))
4160 break
4162 if not found:
4163 sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
4165 def findLastP4Revision(self, starting_point):
4166 """ Look back from starting_point for the first commit created by git-p4
4167 to find the P4 commit we are based on, and the depot-paths.
4170 for parent in (range(65535)):
4171 log = extractLogMessageFromGitCommit("{0}~{1}".format(starting_point, parent))
4172 settings = extractSettingsGitLog(log)
4173 if 'change' in settings:
4174 return settings
4176 sys.exit("could not find git-p4 commits in {0}".format(self.origin))
4178 def createShelveParent(self, change, branch_name, sync, origin):
4179 """ Create a commit matching the parent of the shelved changelist 'change'
4181 parent_description = p4_describe(change, shelved=True)
4182 parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
4183 files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
4185 parent_files = []
4186 for f in files:
4187 # if it was added in the shelved changelist, it won't exist in the parent
4188 if f['action'] in self.add_actions:
4189 continue
4191 # if it was deleted in the shelved changelist it must not be deleted
4192 # in the parent - we might even need to create it if the origin branch
4193 # does not have it
4194 if f['action'] in self.delete_actions:
4195 f['action'] = 'add'
4197 parent_files.append(f)
4199 sync.commit(parent_description, parent_files, branch_name,
4200 parent=origin, allow_empty=True)
4201 print("created parent commit for {0} based on {1} in {2}".format(
4202 change, self.origin, branch_name))
4204 def run(self, args):
4205 if len(args) != 1:
4206 return False
4208 if not gitBranchExists(self.origin):
4209 sys.exit("origin branch {0} does not exist".format(self.origin))
4211 sync = P4Sync()
4212 changes = args
4214 # only one change at a time
4215 change = changes[0]
4217 # if the target branch already exists, rename it
4218 branch_name = "{0}/{1}".format(self.destbranch, change)
4219 if gitBranchExists(branch_name):
4220 self.renameBranch(branch_name)
4221 sync.branch = branch_name
4223 sync.verbose = self.verbose
4224 sync.suppress_meta_comment = True
4226 settings = self.findLastP4Revision(self.origin)
4227 sync.depotPaths = settings['depot-paths']
4228 sync.branchPrefixes = sync.depotPaths
4230 sync.openStreams()
4231 sync.loadUserMapFromCache()
4232 sync.silent = True
4234 # create a commit for the parent of the shelved changelist
4235 self.createShelveParent(change, branch_name, sync, self.origin)
4237 # create the commit for the shelved changelist itself
4238 description = p4_describe(change, True)
4239 files = sync.extractFilesFromCommit(description, True, change)
4241 sync.commit(description, files, branch_name, "")
4242 sync.closeStreams()
4244 print("unshelved changelist {0} into {1}".format(change, branch_name))
4246 return True
4248 class P4Branches(Command):
4249 def __init__(self):
4250 Command.__init__(self)
4251 self.options = [ ]
4252 self.description = ("Shows the git branches that hold imports and their "
4253 + "corresponding perforce depot paths")
4254 self.verbose = False
4256 def run(self, args):
4257 if originP4BranchesExist():
4258 createOrUpdateBranchesFromOrigin()
4260 cmdline = "git rev-parse --symbolic "
4261 cmdline += " --remotes"
4263 for line in read_pipe_lines(cmdline):
4264 line = line.strip()
4266 if not line.startswith('p4/') or line == "p4/HEAD":
4267 continue
4268 branch = line
4270 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4271 settings = extractSettingsGitLog(log)
4273 print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4274 return True
4276 class HelpFormatter(optparse.IndentedHelpFormatter):
4277 def __init__(self):
4278 optparse.IndentedHelpFormatter.__init__(self)
4280 def format_description(self, description):
4281 if description:
4282 return description + "\n"
4283 else:
4284 return ""
4286 def printUsage(commands):
4287 print("usage: %s <command> [options]" % sys.argv[0])
4288 print("")
4289 print("valid commands: %s" % ", ".join(commands))
4290 print("")
4291 print("Try %s <command> --help for command specific help." % sys.argv[0])
4292 print("")
4294 commands = {
4295 "submit" : P4Submit,
4296 "commit" : P4Submit,
4297 "sync" : P4Sync,
4298 "rebase" : P4Rebase,
4299 "clone" : P4Clone,
4300 "branches" : P4Branches,
4301 "unshelve" : P4Unshelve,
4304 def main():
4305 if len(sys.argv[1:]) == 0:
4306 printUsage(commands.keys())
4307 sys.exit(2)
4309 cmdName = sys.argv[1]
4310 try:
4311 klass = commands[cmdName]
4312 cmd = klass()
4313 except KeyError:
4314 print("unknown command %s" % cmdName)
4315 print("")
4316 printUsage(commands.keys())
4317 sys.exit(2)
4319 options = cmd.options
4320 cmd.gitdir = os.environ.get("GIT_DIR", None)
4322 args = sys.argv[2:]
4324 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4325 if cmd.needsGit:
4326 options.append(optparse.make_option("--git-dir", dest="gitdir"))
4328 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4329 options,
4330 description = cmd.description,
4331 formatter = HelpFormatter())
4333 try:
4334 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4335 except:
4336 parser.print_help()
4337 raise
4339 global verbose
4340 verbose = cmd.verbose
4341 if cmd.needsGit:
4342 if cmd.gitdir == None:
4343 cmd.gitdir = os.path.abspath(".git")
4344 if not isValidGitDir(cmd.gitdir):
4345 # "rev-parse --git-dir" without arguments will try $PWD/.git
4346 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4347 if os.path.exists(cmd.gitdir):
4348 cdup = read_pipe("git rev-parse --show-cdup").strip()
4349 if len(cdup) > 0:
4350 chdir(cdup);
4352 if not isValidGitDir(cmd.gitdir):
4353 if isValidGitDir(cmd.gitdir + "/.git"):
4354 cmd.gitdir += "/.git"
4355 else:
4356 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4358 # so git commands invoked from the P4 workspace will succeed
4359 os.environ["GIT_DIR"] = cmd.gitdir
4361 if not cmd.run(args):
4362 parser.print_help()
4363 sys.exit(2)
4366 if __name__ == '__main__':
4367 main()