list-objects-filter: treat NULL filter_options as "disabled"
[git/raj.git] / git-p4.py
blobb8b2a1679e7336e27d794947e9191cd4b5ec1b57
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 # pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except
11 # pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary
12 # pylint: disable=wrong-import-order,unused-import,too-few-public-methods
13 # pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals
14 # pylint: disable=line-too-long,bad-whitespace,superfluous-parens
15 # pylint: disable=too-many-statements,too-many-instance-attributes
16 # pylint: disable=too-many-branches,too-many-nested-blocks
18 import sys
19 if sys.version_info.major < 3 and sys.version_info.minor < 7:
20 sys.stderr.write("git-p4: requires Python 2.7 or later.\n")
21 sys.exit(1)
22 import os
23 import optparse
24 import functools
25 import marshal
26 import subprocess
27 import tempfile
28 import time
29 import platform
30 import re
31 import shutil
32 import stat
33 import zipfile
34 import zlib
35 import ctypes
36 import errno
37 import glob
39 # On python2.7 where raw_input() and input() are both availble,
40 # we want raw_input's semantics, but aliased to input for python3
41 # compatibility
42 # support basestring in python3
43 try:
44 if raw_input and input:
45 input = raw_input
46 except:
47 pass
49 verbose = False
51 # Only labels/tags matching this will be imported/exported
52 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
54 # The block size is reduced automatically if required
55 defaultBlockSize = 1<<20
57 p4_access_checked = False
59 def p4_build_cmd(cmd):
60 """Build a suitable p4 command line.
62 This consolidates building and returning a p4 command line into one
63 location. It means that hooking into the environment, or other configuration
64 can be done more easily.
65 """
66 real_cmd = ["p4"]
68 user = gitConfig("git-p4.user")
69 if len(user) > 0:
70 real_cmd += ["-u",user]
72 password = gitConfig("git-p4.password")
73 if len(password) > 0:
74 real_cmd += ["-P", password]
76 port = gitConfig("git-p4.port")
77 if len(port) > 0:
78 real_cmd += ["-p", port]
80 host = gitConfig("git-p4.host")
81 if len(host) > 0:
82 real_cmd += ["-H", host]
84 client = gitConfig("git-p4.client")
85 if len(client) > 0:
86 real_cmd += ["-c", client]
88 retries = gitConfigInt("git-p4.retries")
89 if retries is None:
90 # Perform 3 retries by default
91 retries = 3
92 if retries > 0:
93 # Provide a way to not pass this option by setting git-p4.retries to 0
94 real_cmd += ["-r", str(retries)]
96 if not isinstance(cmd, list):
97 real_cmd = ' '.join(real_cmd) + ' ' + cmd
98 else:
99 real_cmd += cmd
101 # now check that we can actually talk to the server
102 global p4_access_checked
103 if not p4_access_checked:
104 p4_access_checked = True # suppress access checks in p4_check_access itself
105 p4_check_access()
107 return real_cmd
109 def git_dir(path):
110 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
111 This won't automatically add ".git" to a directory.
113 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
114 if not d or len(d) == 0:
115 return None
116 else:
117 return d
119 def chdir(path, is_client_path=False):
120 """Do chdir to the given path, and set the PWD environment
121 variable for use by P4. It does not look at getcwd() output.
122 Since we're not using the shell, it is necessary to set the
123 PWD environment variable explicitly.
125 Normally, expand the path to force it to be absolute. This
126 addresses the use of relative path names inside P4 settings,
127 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
128 as given; it looks for .p4config using PWD.
130 If is_client_path, the path was handed to us directly by p4,
131 and may be a symbolic link. Do not call os.getcwd() in this
132 case, because it will cause p4 to think that PWD is not inside
133 the client path.
136 os.chdir(path)
137 if not is_client_path:
138 path = os.getcwd()
139 os.environ['PWD'] = path
141 def calcDiskFree():
142 """Return free space in bytes on the disk of the given dirname."""
143 if platform.system() == 'Windows':
144 free_bytes = ctypes.c_ulonglong(0)
145 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
146 return free_bytes.value
147 else:
148 st = os.statvfs(os.getcwd())
149 return st.f_bavail * st.f_frsize
151 def die(msg):
152 """ Terminate execution. Make sure that any running child processes have been wait()ed for before
153 calling this.
155 if verbose:
156 raise Exception(msg)
157 else:
158 sys.stderr.write(msg + "\n")
159 sys.exit(1)
161 def prompt(prompt_text):
162 """ Prompt the user to choose one of the choices
164 Choices are identified in the prompt_text by square brackets around
165 a single letter option.
167 choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
168 while True:
169 sys.stderr.flush()
170 sys.stdout.write(prompt_text)
171 sys.stdout.flush()
172 response=sys.stdin.readline().strip().lower()
173 if not response:
174 continue
175 response = response[0]
176 if response in choices:
177 return response
179 # We need different encoding/decoding strategies for text data being passed
180 # around in pipes depending on python version
181 if bytes is not str:
182 # For python3, always encode and decode as appropriate
183 def decode_text_stream(s):
184 return s.decode() if isinstance(s, bytes) else s
185 def encode_text_stream(s):
186 return s.encode() if isinstance(s, str) else s
187 else:
188 # For python2.7, pass read strings as-is, but also allow writing unicode
189 def decode_text_stream(s):
190 return s
191 def encode_text_stream(s):
192 return s.encode('utf_8') if isinstance(s, unicode) else s
194 def decode_path(path):
195 """Decode a given string (bytes or otherwise) using configured path encoding options
197 encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'
198 if bytes is not str:
199 return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path
200 else:
201 try:
202 path.decode('ascii')
203 except:
204 path = path.decode(encoding, errors='replace')
205 if verbose:
206 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))
207 return path
209 def run_git_hook(cmd, param=[]):
210 """Execute a hook if the hook exists."""
211 if verbose:
212 sys.stderr.write("Looking for hook: %s\n" % cmd)
213 sys.stderr.flush()
215 hooks_path = gitConfig("core.hooksPath")
216 if len(hooks_path) <= 0:
217 hooks_path = os.path.join(os.environ["GIT_DIR"], "hooks")
219 if not isinstance(param, list):
220 param=[param]
222 # resolve hook file name, OS depdenent
223 hook_file = os.path.join(hooks_path, cmd)
224 if platform.system() == 'Windows':
225 if not os.path.isfile(hook_file):
226 # look for the file with an extension
227 files = glob.glob(hook_file + ".*")
228 if not files:
229 return True
230 files.sort()
231 hook_file = files.pop()
232 while hook_file.upper().endswith(".SAMPLE"):
233 # The file is a sample hook. We don't want it
234 if len(files) > 0:
235 hook_file = files.pop()
236 else:
237 return True
239 if not os.path.isfile(hook_file) or not os.access(hook_file, os.X_OK):
240 return True
242 return run_hook_command(hook_file, param) == 0
244 def run_hook_command(cmd, param):
245 """Executes a git hook command
246 cmd = the command line file to be executed. This can be
247 a file that is run by OS association.
249 param = a list of parameters to pass to the cmd command
251 On windows, the extension is checked to see if it should
252 be run with the Git for Windows Bash shell. If there
253 is no file extension, the file is deemed a bash shell
254 and will be handed off to sh.exe. Otherwise, Windows
255 will be called with the shell to handle the file assocation.
257 For non Windows operating systems, the file is called
258 as an executable.
260 cli = [cmd] + param
261 use_shell = False
262 if platform.system() == 'Windows':
263 (root,ext) = os.path.splitext(cmd)
264 if ext == "":
265 exe_path = os.environ.get("EXEPATH")
266 if exe_path is None:
267 exe_path = ""
268 else:
269 exe_path = os.path.join(exe_path, "bin")
270 cli = [os.path.join(exe_path, "SH.EXE")] + cli
271 else:
272 use_shell = True
273 return subprocess.call(cli, shell=use_shell)
276 def write_pipe(c, stdin):
277 if verbose:
278 sys.stderr.write('Writing pipe: %s\n' % str(c))
280 expand = not isinstance(c, list)
281 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
282 pipe = p.stdin
283 val = pipe.write(stdin)
284 pipe.close()
285 if p.wait():
286 die('Command failed: %s' % str(c))
288 return val
290 def p4_write_pipe(c, stdin):
291 real_cmd = p4_build_cmd(c)
292 if bytes is not str and isinstance(stdin, str):
293 stdin = encode_text_stream(stdin)
294 return write_pipe(real_cmd, stdin)
296 def read_pipe_full(c):
297 """ Read output from command. Returns a tuple
298 of the return status, stdout text and stderr
299 text.
301 if verbose:
302 sys.stderr.write('Reading pipe: %s\n' % str(c))
304 expand = not isinstance(c, list)
305 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
306 (out, err) = p.communicate()
307 return (p.returncode, out, decode_text_stream(err))
309 def read_pipe(c, ignore_error=False, raw=False):
310 """ Read output from command. Returns the output text on
311 success. On failure, terminates execution, unless
312 ignore_error is True, when it returns an empty string.
314 If raw is True, do not attempt to decode output text.
316 (retcode, out, err) = read_pipe_full(c)
317 if retcode != 0:
318 if ignore_error:
319 out = ""
320 else:
321 die('Command failed: %s\nError: %s' % (str(c), err))
322 if not raw:
323 out = decode_text_stream(out)
324 return out
326 def read_pipe_text(c):
327 """ Read output from a command with trailing whitespace stripped.
328 On error, returns None.
330 (retcode, out, err) = read_pipe_full(c)
331 if retcode != 0:
332 return None
333 else:
334 return decode_text_stream(out).rstrip()
336 def p4_read_pipe(c, ignore_error=False, raw=False):
337 real_cmd = p4_build_cmd(c)
338 return read_pipe(real_cmd, ignore_error, raw=raw)
340 def read_pipe_lines(c):
341 if verbose:
342 sys.stderr.write('Reading pipe: %s\n' % str(c))
344 expand = not isinstance(c, list)
345 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
346 pipe = p.stdout
347 val = [decode_text_stream(line) for line in pipe.readlines()]
348 if pipe.close() or p.wait():
349 die('Command failed: %s' % str(c))
350 return val
352 def p4_read_pipe_lines(c):
353 """Specifically invoke p4 on the command supplied. """
354 real_cmd = p4_build_cmd(c)
355 return read_pipe_lines(real_cmd)
357 def p4_has_command(cmd):
358 """Ask p4 for help on this command. If it returns an error, the
359 command does not exist in this version of p4."""
360 real_cmd = p4_build_cmd(["help", cmd])
361 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
362 stderr=subprocess.PIPE)
363 p.communicate()
364 return p.returncode == 0
366 def p4_has_move_command():
367 """See if the move command exists, that it supports -k, and that
368 it has not been administratively disabled. The arguments
369 must be correct, but the filenames do not have to exist. Use
370 ones with wildcards so even if they exist, it will fail."""
372 if not p4_has_command("move"):
373 return False
374 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
375 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
376 (out, err) = p.communicate()
377 err = decode_text_stream(err)
378 # return code will be 1 in either case
379 if err.find("Invalid option") >= 0:
380 return False
381 if err.find("disabled") >= 0:
382 return False
383 # assume it failed because @... was invalid changelist
384 return True
386 def system(cmd, ignore_error=False):
387 expand = not isinstance(cmd, list)
388 if verbose:
389 sys.stderr.write("executing %s\n" % str(cmd))
390 retcode = subprocess.call(cmd, shell=expand)
391 if retcode and not ignore_error:
392 raise CalledProcessError(retcode, cmd)
394 return retcode
396 def p4_system(cmd):
397 """Specifically invoke p4 as the system command. """
398 real_cmd = p4_build_cmd(cmd)
399 expand = not isinstance(real_cmd, list)
400 retcode = subprocess.call(real_cmd, shell=expand)
401 if retcode:
402 raise CalledProcessError(retcode, real_cmd)
404 def die_bad_access(s):
405 die("failure accessing depot: {0}".format(s.rstrip()))
407 def p4_check_access(min_expiration=1):
408 """ Check if we can access Perforce - account still logged in
410 results = p4CmdList(["login", "-s"])
412 if len(results) == 0:
413 # should never get here: always get either some results, or a p4ExitCode
414 assert("could not parse response from perforce")
416 result = results[0]
418 if 'p4ExitCode' in result:
419 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
420 die_bad_access("could not run p4")
422 code = result.get("code")
423 if not code:
424 # we get here if we couldn't connect and there was nothing to unmarshal
425 die_bad_access("could not connect")
427 elif code == "stat":
428 expiry = result.get("TicketExpiration")
429 if expiry:
430 expiry = int(expiry)
431 if expiry > min_expiration:
432 # ok to carry on
433 return
434 else:
435 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
437 else:
438 # account without a timeout - all ok
439 return
441 elif code == "error":
442 data = result.get("data")
443 if data:
444 die_bad_access("p4 error: {0}".format(data))
445 else:
446 die_bad_access("unknown error")
447 elif code == "info":
448 return
449 else:
450 die_bad_access("unknown error code {0}".format(code))
452 _p4_version_string = None
453 def p4_version_string():
454 """Read the version string, showing just the last line, which
455 hopefully is the interesting version bit.
457 $ p4 -V
458 Perforce - The Fast Software Configuration Management System.
459 Copyright 1995-2011 Perforce Software. All rights reserved.
460 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
462 global _p4_version_string
463 if not _p4_version_string:
464 a = p4_read_pipe_lines(["-V"])
465 _p4_version_string = a[-1].rstrip()
466 return _p4_version_string
468 def p4_integrate(src, dest):
469 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
471 def p4_sync(f, *options):
472 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
474 def p4_add(f):
475 # forcibly add file names with wildcards
476 if wildcard_present(f):
477 p4_system(["add", "-f", f])
478 else:
479 p4_system(["add", f])
481 def p4_delete(f):
482 p4_system(["delete", wildcard_encode(f)])
484 def p4_edit(f, *options):
485 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
487 def p4_revert(f):
488 p4_system(["revert", wildcard_encode(f)])
490 def p4_reopen(type, f):
491 p4_system(["reopen", "-t", type, wildcard_encode(f)])
493 def p4_reopen_in_change(changelist, files):
494 cmd = ["reopen", "-c", str(changelist)] + files
495 p4_system(cmd)
497 def p4_move(src, dest):
498 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
500 def p4_last_change():
501 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
502 return int(results[0]['change'])
504 def p4_describe(change, shelved=False):
505 """Make sure it returns a valid result by checking for
506 the presence of field "time". Return a dict of the
507 results."""
509 cmd = ["describe", "-s"]
510 if shelved:
511 cmd += ["-S"]
512 cmd += [str(change)]
514 ds = p4CmdList(cmd, skip_info=True)
515 if len(ds) != 1:
516 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
518 d = ds[0]
520 if "p4ExitCode" in d:
521 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
522 str(d)))
523 if "code" in d:
524 if d["code"] == "error":
525 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
527 if "time" not in d:
528 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
530 return d
533 # Canonicalize the p4 type and return a tuple of the
534 # base type, plus any modifiers. See "p4 help filetypes"
535 # for a list and explanation.
537 def split_p4_type(p4type):
539 p4_filetypes_historical = {
540 "ctempobj": "binary+Sw",
541 "ctext": "text+C",
542 "cxtext": "text+Cx",
543 "ktext": "text+k",
544 "kxtext": "text+kx",
545 "ltext": "text+F",
546 "tempobj": "binary+FSw",
547 "ubinary": "binary+F",
548 "uresource": "resource+F",
549 "uxbinary": "binary+Fx",
550 "xbinary": "binary+x",
551 "xltext": "text+Fx",
552 "xtempobj": "binary+Swx",
553 "xtext": "text+x",
554 "xunicode": "unicode+x",
555 "xutf16": "utf16+x",
557 if p4type in p4_filetypes_historical:
558 p4type = p4_filetypes_historical[p4type]
559 mods = ""
560 s = p4type.split("+")
561 base = s[0]
562 mods = ""
563 if len(s) > 1:
564 mods = s[1]
565 return (base, mods)
568 # return the raw p4 type of a file (text, text+ko, etc)
570 def p4_type(f):
571 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
572 return results[0]['headType']
575 # Given a type base and modifier, return a regexp matching
576 # the keywords that can be expanded in the file
578 def p4_keywords_regexp_for_type(base, type_mods):
579 if base in ("text", "unicode", "binary"):
580 kwords = None
581 if "ko" in type_mods:
582 kwords = 'Id|Header'
583 elif "k" in type_mods:
584 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
585 else:
586 return None
587 pattern = r"""
588 \$ # Starts with a dollar, followed by...
589 (%s) # one of the keywords, followed by...
590 (:[^$\n]+)? # possibly an old expansion, followed by...
591 \$ # another dollar
592 """ % kwords
593 return pattern
594 else:
595 return None
598 # Given a file, return a regexp matching the possible
599 # RCS keywords that will be expanded, or None for files
600 # with kw expansion turned off.
602 def p4_keywords_regexp_for_file(file):
603 if not os.path.exists(file):
604 return None
605 else:
606 (type_base, type_mods) = split_p4_type(p4_type(file))
607 return p4_keywords_regexp_for_type(type_base, type_mods)
609 def setP4ExecBit(file, mode):
610 # Reopens an already open file and changes the execute bit to match
611 # the execute bit setting in the passed in mode.
613 p4Type = "+x"
615 if not isModeExec(mode):
616 p4Type = getP4OpenedType(file)
617 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
618 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
619 if p4Type[-1] == "+":
620 p4Type = p4Type[0:-1]
622 p4_reopen(p4Type, file)
624 def getP4OpenedType(file):
625 # Returns the perforce file type for the given file.
627 result = p4_read_pipe(["opened", wildcard_encode(file)])
628 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
629 if match:
630 return match.group(1)
631 else:
632 die("Could not determine file type for %s (result: '%s')" % (file, result))
634 # Return the set of all p4 labels
635 def getP4Labels(depotPaths):
636 labels = set()
637 if not isinstance(depotPaths, list):
638 depotPaths = [depotPaths]
640 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
641 label = l['label']
642 labels.add(label)
644 return labels
646 # Return the set of all git tags
647 def getGitTags():
648 gitTags = set()
649 for line in read_pipe_lines(["git", "tag"]):
650 tag = line.strip()
651 gitTags.add(tag)
652 return gitTags
654 _diff_tree_pattern = None
656 def parseDiffTreeEntry(entry):
657 """Parses a single diff tree entry into its component elements.
659 See git-diff-tree(1) manpage for details about the format of the diff
660 output. This method returns a dictionary with the following elements:
662 src_mode - The mode of the source file
663 dst_mode - The mode of the destination file
664 src_sha1 - The sha1 for the source file
665 dst_sha1 - The sha1 fr the destination file
666 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
667 status_score - The score for the status (applicable for 'C' and 'R'
668 statuses). This is None if there is no score.
669 src - The path for the source file.
670 dst - The path for the destination file. This is only present for
671 copy or renames. If it is not present, this is None.
673 If the pattern is not matched, None is returned."""
675 global _diff_tree_pattern
676 if not _diff_tree_pattern:
677 _diff_tree_pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
679 match = _diff_tree_pattern.match(entry)
680 if match:
681 return {
682 'src_mode': match.group(1),
683 'dst_mode': match.group(2),
684 'src_sha1': match.group(3),
685 'dst_sha1': match.group(4),
686 'status': match.group(5),
687 'status_score': match.group(6),
688 'src': match.group(7),
689 'dst': match.group(10)
691 return None
693 def isModeExec(mode):
694 # Returns True if the given git mode represents an executable file,
695 # otherwise False.
696 return mode[-3:] == "755"
698 class P4Exception(Exception):
699 """ Base class for exceptions from the p4 client """
700 def __init__(self, exit_code):
701 self.p4ExitCode = exit_code
703 class P4ServerException(P4Exception):
704 """ Base class for exceptions where we get some kind of marshalled up result from the server """
705 def __init__(self, exit_code, p4_result):
706 super(P4ServerException, self).__init__(exit_code)
707 self.p4_result = p4_result
708 self.code = p4_result[0]['code']
709 self.data = p4_result[0]['data']
711 class P4RequestSizeException(P4ServerException):
712 """ One of the maxresults or maxscanrows errors """
713 def __init__(self, exit_code, p4_result, limit):
714 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
715 self.limit = limit
717 class P4CommandException(P4Exception):
718 """ Something went wrong calling p4 which means we have to give up """
719 def __init__(self, msg):
720 self.msg = msg
722 def __str__(self):
723 return self.msg
725 def isModeExecChanged(src_mode, dst_mode):
726 return isModeExec(src_mode) != isModeExec(dst_mode)
728 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
729 errors_as_exceptions=False):
731 if not isinstance(cmd, list):
732 cmd = "-G " + cmd
733 expand = True
734 else:
735 cmd = ["-G"] + cmd
736 expand = False
738 cmd = p4_build_cmd(cmd)
739 if verbose:
740 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
742 # Use a temporary file to avoid deadlocks without
743 # subprocess.communicate(), which would put another copy
744 # of stdout into memory.
745 stdin_file = None
746 if stdin is not None:
747 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
748 if not isinstance(stdin, list):
749 stdin_file.write(stdin)
750 else:
751 for i in stdin:
752 stdin_file.write(encode_text_stream(i))
753 stdin_file.write(b'\n')
754 stdin_file.flush()
755 stdin_file.seek(0)
757 p4 = subprocess.Popen(cmd,
758 shell=expand,
759 stdin=stdin_file,
760 stdout=subprocess.PIPE)
762 result = []
763 try:
764 while True:
765 entry = marshal.load(p4.stdout)
766 if bytes is not str:
767 # Decode unmarshalled dict to use str keys and values, except for:
768 # - `data` which may contain arbitrary binary data
769 # - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
770 decoded_entry = {}
771 for key, value in entry.items():
772 key = key.decode()
773 if isinstance(value, bytes) and not (key in ('data', 'path', 'clientFile') or key.startswith('depotFile')):
774 value = value.decode()
775 decoded_entry[key] = value
776 # Parse out data if it's an error response
777 if decoded_entry.get('code') == 'error' and 'data' in decoded_entry:
778 decoded_entry['data'] = decoded_entry['data'].decode()
779 entry = decoded_entry
780 if skip_info:
781 if 'code' in entry and entry['code'] == 'info':
782 continue
783 if cb is not None:
784 cb(entry)
785 else:
786 result.append(entry)
787 except EOFError:
788 pass
789 exitCode = p4.wait()
790 if exitCode != 0:
791 if errors_as_exceptions:
792 if len(result) > 0:
793 data = result[0].get('data')
794 if data:
795 m = re.search('Too many rows scanned \(over (\d+)\)', data)
796 if not m:
797 m = re.search('Request too large \(over (\d+)\)', data)
799 if m:
800 limit = int(m.group(1))
801 raise P4RequestSizeException(exitCode, result, limit)
803 raise P4ServerException(exitCode, result)
804 else:
805 raise P4Exception(exitCode)
806 else:
807 entry = {}
808 entry["p4ExitCode"] = exitCode
809 result.append(entry)
811 return result
813 def p4Cmd(cmd):
814 list = p4CmdList(cmd)
815 result = {}
816 for entry in list:
817 result.update(entry)
818 return result;
820 def p4Where(depotPath):
821 if not depotPath.endswith("/"):
822 depotPath += "/"
823 depotPathLong = depotPath + "..."
824 outputList = p4CmdList(["where", depotPathLong])
825 output = None
826 for entry in outputList:
827 if "depotFile" in entry:
828 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
829 # The base path always ends with "/...".
830 entry_path = decode_path(entry['depotFile'])
831 if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":
832 output = entry
833 break
834 elif "data" in entry:
835 data = entry.get("data")
836 space = data.find(" ")
837 if data[:space] == depotPath:
838 output = entry
839 break
840 if output == None:
841 return ""
842 if output["code"] == "error":
843 return ""
844 clientPath = ""
845 if "path" in output:
846 clientPath = decode_path(output['path'])
847 elif "data" in output:
848 data = output.get("data")
849 lastSpace = data.rfind(b" ")
850 clientPath = decode_path(data[lastSpace + 1:])
852 if clientPath.endswith("..."):
853 clientPath = clientPath[:-3]
854 return clientPath
856 def currentGitBranch():
857 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
859 def isValidGitDir(path):
860 return git_dir(path) != None
862 def parseRevision(ref):
863 return read_pipe("git rev-parse %s" % ref).strip()
865 def branchExists(ref):
866 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
867 ignore_error=True)
868 return len(rev) > 0
870 def extractLogMessageFromGitCommit(commit):
871 logMessage = ""
873 ## fixme: title is first line of commit, not 1st paragraph.
874 foundTitle = False
875 for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
876 if not foundTitle:
877 if len(log) == 1:
878 foundTitle = True
879 continue
881 logMessage += log
882 return logMessage
884 def extractSettingsGitLog(log):
885 values = {}
886 for line in log.split("\n"):
887 line = line.strip()
888 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
889 if not m:
890 continue
892 assignments = m.group(1).split (':')
893 for a in assignments:
894 vals = a.split ('=')
895 key = vals[0].strip()
896 val = ('='.join (vals[1:])).strip()
897 if val.endswith ('\"') and val.startswith('"'):
898 val = val[1:-1]
900 values[key] = val
902 paths = values.get("depot-paths")
903 if not paths:
904 paths = values.get("depot-path")
905 if paths:
906 values['depot-paths'] = paths.split(',')
907 return values
909 def gitBranchExists(branch):
910 proc = subprocess.Popen(["git", "rev-parse", branch],
911 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
912 return proc.wait() == 0;
914 def gitUpdateRef(ref, newvalue):
915 subprocess.check_call(["git", "update-ref", ref, newvalue])
917 def gitDeleteRef(ref):
918 subprocess.check_call(["git", "update-ref", "-d", ref])
920 _gitConfig = {}
922 def gitConfig(key, typeSpecifier=None):
923 if key not in _gitConfig:
924 cmd = [ "git", "config" ]
925 if typeSpecifier:
926 cmd += [ typeSpecifier ]
927 cmd += [ key ]
928 s = read_pipe(cmd, ignore_error=True)
929 _gitConfig[key] = s.strip()
930 return _gitConfig[key]
932 def gitConfigBool(key):
933 """Return a bool, using git config --bool. It is True only if the
934 variable is set to true, and False if set to false or not present
935 in the config."""
937 if key not in _gitConfig:
938 _gitConfig[key] = gitConfig(key, '--bool') == "true"
939 return _gitConfig[key]
941 def gitConfigInt(key):
942 if key not in _gitConfig:
943 cmd = [ "git", "config", "--int", key ]
944 s = read_pipe(cmd, ignore_error=True)
945 v = s.strip()
946 try:
947 _gitConfig[key] = int(gitConfig(key, '--int'))
948 except ValueError:
949 _gitConfig[key] = None
950 return _gitConfig[key]
952 def gitConfigList(key):
953 if key not in _gitConfig:
954 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
955 _gitConfig[key] = s.strip().splitlines()
956 if _gitConfig[key] == ['']:
957 _gitConfig[key] = []
958 return _gitConfig[key]
960 def p4BranchesInGit(branchesAreInRemotes=True):
961 """Find all the branches whose names start with "p4/", looking
962 in remotes or heads as specified by the argument. Return
963 a dictionary of { branch: revision } for each one found.
964 The branch names are the short names, without any
965 "p4/" prefix."""
967 branches = {}
969 cmdline = "git rev-parse --symbolic "
970 if branchesAreInRemotes:
971 cmdline += "--remotes"
972 else:
973 cmdline += "--branches"
975 for line in read_pipe_lines(cmdline):
976 line = line.strip()
978 # only import to p4/
979 if not line.startswith('p4/'):
980 continue
981 # special symbolic ref to p4/master
982 if line == "p4/HEAD":
983 continue
985 # strip off p4/ prefix
986 branch = line[len("p4/"):]
988 branches[branch] = parseRevision(line)
990 return branches
992 def branch_exists(branch):
993 """Make sure that the given ref name really exists."""
995 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
996 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
997 out, _ = p.communicate()
998 out = decode_text_stream(out)
999 if p.returncode:
1000 return False
1001 # expect exactly one line of output: the branch name
1002 return out.rstrip() == branch
1004 def findUpstreamBranchPoint(head = "HEAD"):
1005 branches = p4BranchesInGit()
1006 # map from depot-path to branch name
1007 branchByDepotPath = {}
1008 for branch in branches.keys():
1009 tip = branches[branch]
1010 log = extractLogMessageFromGitCommit(tip)
1011 settings = extractSettingsGitLog(log)
1012 if "depot-paths" in settings:
1013 paths = ",".join(settings["depot-paths"])
1014 branchByDepotPath[paths] = "remotes/p4/" + branch
1016 settings = None
1017 parent = 0
1018 while parent < 65535:
1019 commit = head + "~%s" % parent
1020 log = extractLogMessageFromGitCommit(commit)
1021 settings = extractSettingsGitLog(log)
1022 if "depot-paths" in settings:
1023 paths = ",".join(settings["depot-paths"])
1024 if paths in branchByDepotPath:
1025 return [branchByDepotPath[paths], settings]
1027 parent = parent + 1
1029 return ["", settings]
1031 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
1032 if not silent:
1033 print("Creating/updating branch(es) in %s based on origin branch(es)"
1034 % localRefPrefix)
1036 originPrefix = "origin/p4/"
1038 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
1039 line = line.strip()
1040 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
1041 continue
1043 headName = line[len(originPrefix):]
1044 remoteHead = localRefPrefix + headName
1045 originHead = line
1047 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
1048 if ('depot-paths' not in original
1049 or 'change' not in original):
1050 continue
1052 update = False
1053 if not gitBranchExists(remoteHead):
1054 if verbose:
1055 print("creating %s" % remoteHead)
1056 update = True
1057 else:
1058 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
1059 if 'change' in settings:
1060 if settings['depot-paths'] == original['depot-paths']:
1061 originP4Change = int(original['change'])
1062 p4Change = int(settings['change'])
1063 if originP4Change > p4Change:
1064 print("%s (%s) is newer than %s (%s). "
1065 "Updating p4 branch from origin."
1066 % (originHead, originP4Change,
1067 remoteHead, p4Change))
1068 update = True
1069 else:
1070 print("Ignoring: %s was imported from %s while "
1071 "%s was imported from %s"
1072 % (originHead, ','.join(original['depot-paths']),
1073 remoteHead, ','.join(settings['depot-paths'])))
1075 if update:
1076 system("git update-ref %s %s" % (remoteHead, originHead))
1078 def originP4BranchesExist():
1079 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1082 def p4ParseNumericChangeRange(parts):
1083 changeStart = int(parts[0][1:])
1084 if parts[1] == '#head':
1085 changeEnd = p4_last_change()
1086 else:
1087 changeEnd = int(parts[1])
1089 return (changeStart, changeEnd)
1091 def chooseBlockSize(blockSize):
1092 if blockSize:
1093 return blockSize
1094 else:
1095 return defaultBlockSize
1097 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
1098 assert depotPaths
1100 # Parse the change range into start and end. Try to find integer
1101 # revision ranges as these can be broken up into blocks to avoid
1102 # hitting server-side limits (maxrows, maxscanresults). But if
1103 # that doesn't work, fall back to using the raw revision specifier
1104 # strings, without using block mode.
1106 if changeRange is None or changeRange == '':
1107 changeStart = 1
1108 changeEnd = p4_last_change()
1109 block_size = chooseBlockSize(requestedBlockSize)
1110 else:
1111 parts = changeRange.split(',')
1112 assert len(parts) == 2
1113 try:
1114 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
1115 block_size = chooseBlockSize(requestedBlockSize)
1116 except ValueError:
1117 changeStart = parts[0][1:]
1118 changeEnd = parts[1]
1119 if requestedBlockSize:
1120 die("cannot use --changes-block-size with non-numeric revisions")
1121 block_size = None
1123 changes = set()
1125 # Retrieve changes a block at a time, to prevent running
1126 # into a MaxResults/MaxScanRows error from the server. If
1127 # we _do_ hit one of those errors, turn down the block size
1129 while True:
1130 cmd = ['changes']
1132 if block_size:
1133 end = min(changeEnd, changeStart + block_size)
1134 revisionRange = "%d,%d" % (changeStart, end)
1135 else:
1136 revisionRange = "%s,%s" % (changeStart, changeEnd)
1138 for p in depotPaths:
1139 cmd += ["%s...@%s" % (p, revisionRange)]
1141 # fetch the changes
1142 try:
1143 result = p4CmdList(cmd, errors_as_exceptions=True)
1144 except P4RequestSizeException as e:
1145 if not block_size:
1146 block_size = e.limit
1147 elif block_size > e.limit:
1148 block_size = e.limit
1149 else:
1150 block_size = max(2, block_size // 2)
1152 if verbose: print("block size error, retrying with block size {0}".format(block_size))
1153 continue
1154 except P4Exception as e:
1155 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1157 # Insert changes in chronological order
1158 for entry in reversed(result):
1159 if 'change' not in entry:
1160 continue
1161 changes.add(int(entry['change']))
1163 if not block_size:
1164 break
1166 if end >= changeEnd:
1167 break
1169 changeStart = end + 1
1171 changes = sorted(changes)
1172 return changes
1174 def p4PathStartsWith(path, prefix):
1175 # This method tries to remedy a potential mixed-case issue:
1177 # If UserA adds //depot/DirA/file1
1178 # and UserB adds //depot/dira/file2
1180 # we may or may not have a problem. If you have core.ignorecase=true,
1181 # we treat DirA and dira as the same directory
1182 if gitConfigBool("core.ignorecase"):
1183 return path.lower().startswith(prefix.lower())
1184 return path.startswith(prefix)
1186 def getClientSpec():
1187 """Look at the p4 client spec, create a View() object that contains
1188 all the mappings, and return it."""
1190 specList = p4CmdList("client -o")
1191 if len(specList) != 1:
1192 die('Output from "client -o" is %d lines, expecting 1' %
1193 len(specList))
1195 # dictionary of all client parameters
1196 entry = specList[0]
1198 # the //client/ name
1199 client_name = entry["Client"]
1201 # just the keys that start with "View"
1202 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1204 # hold this new View
1205 view = View(client_name)
1207 # append the lines, in order, to the view
1208 for view_num in range(len(view_keys)):
1209 k = "View%d" % view_num
1210 if k not in view_keys:
1211 die("Expected view key %s missing" % k)
1212 view.append(entry[k])
1214 return view
1216 def getClientRoot():
1217 """Grab the client directory."""
1219 output = p4CmdList("client -o")
1220 if len(output) != 1:
1221 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1223 entry = output[0]
1224 if "Root" not in entry:
1225 die('Client has no "Root"')
1227 return entry["Root"]
1230 # P4 wildcards are not allowed in filenames. P4 complains
1231 # if you simply add them, but you can force it with "-f", in
1232 # which case it translates them into %xx encoding internally.
1234 def wildcard_decode(path):
1235 # Search for and fix just these four characters. Do % last so
1236 # that fixing it does not inadvertently create new %-escapes.
1237 # Cannot have * in a filename in windows; untested as to
1238 # what p4 would do in such a case.
1239 if not platform.system() == "Windows":
1240 path = path.replace("%2A", "*")
1241 path = path.replace("%23", "#") \
1242 .replace("%40", "@") \
1243 .replace("%25", "%")
1244 return path
1246 def wildcard_encode(path):
1247 # do % first to avoid double-encoding the %s introduced here
1248 path = path.replace("%", "%25") \
1249 .replace("*", "%2A") \
1250 .replace("#", "%23") \
1251 .replace("@", "%40")
1252 return path
1254 def wildcard_present(path):
1255 m = re.search("[*#@%]", path)
1256 return m is not None
1258 class LargeFileSystem(object):
1259 """Base class for large file system support."""
1261 def __init__(self, writeToGitStream):
1262 self.largeFiles = set()
1263 self.writeToGitStream = writeToGitStream
1265 def generatePointer(self, cloneDestination, contentFile):
1266 """Return the content of a pointer file that is stored in Git instead of
1267 the actual content."""
1268 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1270 def pushFile(self, localLargeFile):
1271 """Push the actual content which is not stored in the Git repository to
1272 a server."""
1273 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1275 def hasLargeFileExtension(self, relPath):
1276 return functools.reduce(
1277 lambda a, b: a or b,
1278 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1279 False
1282 def generateTempFile(self, contents):
1283 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1284 for d in contents:
1285 contentFile.write(d)
1286 contentFile.close()
1287 return contentFile.name
1289 def exceedsLargeFileThreshold(self, relPath, contents):
1290 if gitConfigInt('git-p4.largeFileThreshold'):
1291 contentsSize = sum(len(d) for d in contents)
1292 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1293 return True
1294 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1295 contentsSize = sum(len(d) for d in contents)
1296 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1297 return False
1298 contentTempFile = self.generateTempFile(contents)
1299 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=True)
1300 with zipfile.ZipFile(compressedContentFile, mode='w') as zf:
1301 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1302 compressedContentsSize = zf.infolist()[0].compress_size
1303 os.remove(contentTempFile)
1304 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1305 return True
1306 return False
1308 def addLargeFile(self, relPath):
1309 self.largeFiles.add(relPath)
1311 def removeLargeFile(self, relPath):
1312 self.largeFiles.remove(relPath)
1314 def isLargeFile(self, relPath):
1315 return relPath in self.largeFiles
1317 def processContent(self, git_mode, relPath, contents):
1318 """Processes the content of git fast import. This method decides if a
1319 file is stored in the large file system and handles all necessary
1320 steps."""
1321 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1322 contentTempFile = self.generateTempFile(contents)
1323 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1324 if pointer_git_mode:
1325 git_mode = pointer_git_mode
1326 if localLargeFile:
1327 # Move temp file to final location in large file system
1328 largeFileDir = os.path.dirname(localLargeFile)
1329 if not os.path.isdir(largeFileDir):
1330 os.makedirs(largeFileDir)
1331 shutil.move(contentTempFile, localLargeFile)
1332 self.addLargeFile(relPath)
1333 if gitConfigBool('git-p4.largeFilePush'):
1334 self.pushFile(localLargeFile)
1335 if verbose:
1336 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1337 return (git_mode, contents)
1339 class MockLFS(LargeFileSystem):
1340 """Mock large file system for testing."""
1342 def generatePointer(self, contentFile):
1343 """The pointer content is the original content prefixed with "pointer-".
1344 The local filename of the large file storage is derived from the file content.
1346 with open(contentFile, 'r') as f:
1347 content = next(f)
1348 gitMode = '100644'
1349 pointerContents = 'pointer-' + content
1350 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1351 return (gitMode, pointerContents, localLargeFile)
1353 def pushFile(self, localLargeFile):
1354 """The remote filename of the large file storage is the same as the local
1355 one but in a different directory.
1357 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1358 if not os.path.exists(remotePath):
1359 os.makedirs(remotePath)
1360 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1362 class GitLFS(LargeFileSystem):
1363 """Git LFS as backend for the git-p4 large file system.
1364 See https://git-lfs.github.com/ for details."""
1366 def __init__(self, *args):
1367 LargeFileSystem.__init__(self, *args)
1368 self.baseGitAttributes = []
1370 def generatePointer(self, contentFile):
1371 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1372 mode and content which is stored in the Git repository instead of
1373 the actual content. Return also the new location of the actual
1374 content.
1376 if os.path.getsize(contentFile) == 0:
1377 return (None, '', None)
1379 pointerProcess = subprocess.Popen(
1380 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1381 stdout=subprocess.PIPE
1383 pointerFile = decode_text_stream(pointerProcess.stdout.read())
1384 if pointerProcess.wait():
1385 os.remove(contentFile)
1386 die('git-lfs pointer command failed. Did you install the extension?')
1388 # Git LFS removed the preamble in the output of the 'pointer' command
1389 # starting from version 1.2.0. Check for the preamble here to support
1390 # earlier versions.
1391 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1392 if pointerFile.startswith('Git LFS pointer for'):
1393 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1395 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1396 # if someone use external lfs.storage ( not in local repo git )
1397 lfs_path = gitConfig('lfs.storage')
1398 if not lfs_path:
1399 lfs_path = 'lfs'
1400 if not os.path.isabs(lfs_path):
1401 lfs_path = os.path.join(os.getcwd(), '.git', lfs_path)
1402 localLargeFile = os.path.join(
1403 lfs_path,
1404 'objects', oid[:2], oid[2:4],
1405 oid,
1407 # LFS Spec states that pointer files should not have the executable bit set.
1408 gitMode = '100644'
1409 return (gitMode, pointerFile, localLargeFile)
1411 def pushFile(self, localLargeFile):
1412 uploadProcess = subprocess.Popen(
1413 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1415 if uploadProcess.wait():
1416 die('git-lfs push command failed. Did you define a remote?')
1418 def generateGitAttributes(self):
1419 return (
1420 self.baseGitAttributes +
1422 '\n',
1423 '#\n',
1424 '# Git LFS (see https://git-lfs.github.com/)\n',
1425 '#\n',
1427 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1428 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1430 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1431 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1435 def addLargeFile(self, relPath):
1436 LargeFileSystem.addLargeFile(self, relPath)
1437 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1439 def removeLargeFile(self, relPath):
1440 LargeFileSystem.removeLargeFile(self, relPath)
1441 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1443 def processContent(self, git_mode, relPath, contents):
1444 if relPath == '.gitattributes':
1445 self.baseGitAttributes = contents
1446 return (git_mode, self.generateGitAttributes())
1447 else:
1448 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1450 class Command:
1451 delete_actions = ( "delete", "move/delete", "purge" )
1452 add_actions = ( "add", "branch", "move/add" )
1454 def __init__(self):
1455 self.usage = "usage: %prog [options]"
1456 self.needsGit = True
1457 self.verbose = False
1459 # This is required for the "append" update_shelve action
1460 def ensure_value(self, attr, value):
1461 if not hasattr(self, attr) or getattr(self, attr) is None:
1462 setattr(self, attr, value)
1463 return getattr(self, attr)
1465 class P4UserMap:
1466 def __init__(self):
1467 self.userMapFromPerforceServer = False
1468 self.myP4UserId = None
1470 def p4UserId(self):
1471 if self.myP4UserId:
1472 return self.myP4UserId
1474 results = p4CmdList("user -o")
1475 for r in results:
1476 if 'User' in r:
1477 self.myP4UserId = r['User']
1478 return r['User']
1479 die("Could not find your p4 user id")
1481 def p4UserIsMe(self, p4User):
1482 # return True if the given p4 user is actually me
1483 me = self.p4UserId()
1484 if not p4User or p4User != me:
1485 return False
1486 else:
1487 return True
1489 def getUserCacheFilename(self):
1490 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1491 return home + "/.gitp4-usercache.txt"
1493 def getUserMapFromPerforceServer(self):
1494 if self.userMapFromPerforceServer:
1495 return
1496 self.users = {}
1497 self.emails = {}
1499 for output in p4CmdList("users"):
1500 if "User" not in output:
1501 continue
1502 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1503 self.emails[output["Email"]] = output["User"]
1505 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1506 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1507 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1508 if mapUser and len(mapUser[0]) == 3:
1509 user = mapUser[0][0]
1510 fullname = mapUser[0][1]
1511 email = mapUser[0][2]
1512 self.users[user] = fullname + " <" + email + ">"
1513 self.emails[email] = user
1515 s = ''
1516 for (key, val) in self.users.items():
1517 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1519 open(self.getUserCacheFilename(), 'w').write(s)
1520 self.userMapFromPerforceServer = True
1522 def loadUserMapFromCache(self):
1523 self.users = {}
1524 self.userMapFromPerforceServer = False
1525 try:
1526 cache = open(self.getUserCacheFilename(), 'r')
1527 lines = cache.readlines()
1528 cache.close()
1529 for line in lines:
1530 entry = line.strip().split("\t")
1531 self.users[entry[0]] = entry[1]
1532 except IOError:
1533 self.getUserMapFromPerforceServer()
1535 class P4Debug(Command):
1536 def __init__(self):
1537 Command.__init__(self)
1538 self.options = []
1539 self.description = "A tool to debug the output of p4 -G."
1540 self.needsGit = False
1542 def run(self, args):
1543 j = 0
1544 for output in p4CmdList(args):
1545 print('Element: %d' % j)
1546 j += 1
1547 print(output)
1548 return True
1550 class P4RollBack(Command):
1551 def __init__(self):
1552 Command.__init__(self)
1553 self.options = [
1554 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1556 self.description = "A tool to debug the multi-branch import. Don't use :)"
1557 self.rollbackLocalBranches = False
1559 def run(self, args):
1560 if len(args) != 1:
1561 return False
1562 maxChange = int(args[0])
1564 if "p4ExitCode" in p4Cmd("changes -m 1"):
1565 die("Problems executing p4");
1567 if self.rollbackLocalBranches:
1568 refPrefix = "refs/heads/"
1569 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1570 else:
1571 refPrefix = "refs/remotes/"
1572 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1574 for line in lines:
1575 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1576 line = line.strip()
1577 ref = refPrefix + line
1578 log = extractLogMessageFromGitCommit(ref)
1579 settings = extractSettingsGitLog(log)
1581 depotPaths = settings['depot-paths']
1582 change = settings['change']
1584 changed = False
1586 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1587 for p in depotPaths]))) == 0:
1588 print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
1589 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1590 continue
1592 while change and int(change) > maxChange:
1593 changed = True
1594 if self.verbose:
1595 print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
1596 system("git update-ref %s \"%s^\"" % (ref, ref))
1597 log = extractLogMessageFromGitCommit(ref)
1598 settings = extractSettingsGitLog(log)
1601 depotPaths = settings['depot-paths']
1602 change = settings['change']
1604 if changed:
1605 print("%s rewound to %s" % (ref, change))
1607 return True
1609 class P4Submit(Command, P4UserMap):
1611 conflict_behavior_choices = ("ask", "skip", "quit")
1613 def __init__(self):
1614 Command.__init__(self)
1615 P4UserMap.__init__(self)
1616 self.options = [
1617 optparse.make_option("--origin", dest="origin"),
1618 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1619 # preserve the user, requires relevant p4 permissions
1620 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1621 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1622 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1623 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1624 optparse.make_option("--conflict", dest="conflict_behavior",
1625 choices=self.conflict_behavior_choices),
1626 optparse.make_option("--branch", dest="branch"),
1627 optparse.make_option("--shelve", dest="shelve", action="store_true",
1628 help="Shelve instead of submit. Shelved files are reverted, "
1629 "restoring the workspace to the state before the shelve"),
1630 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1631 metavar="CHANGELIST",
1632 help="update an existing shelved changelist, implies --shelve, "
1633 "repeat in-order for multiple shelved changelists"),
1634 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1635 help="submit only the specified commit(s), one commit or xxx..xxx"),
1636 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1637 help="Disable rebase after submit is completed. Can be useful if you "
1638 "work from a local git branch that is not master"),
1639 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1640 help="Skip Perforce sync of p4/master after submit or shelve"),
1641 optparse.make_option("--no-verify", dest="no_verify", action="store_true",
1642 help="Bypass p4-pre-submit and p4-changelist hooks"),
1644 self.description = """Submit changes from git to the perforce depot.\n
1645 The `p4-pre-submit` hook is executed if it exists and is executable. It
1646 can be bypassed with the `--no-verify` command line option. The hook takes
1647 no parameters and nothing from standard input. Exiting with a non-zero status
1648 from this script prevents `git-p4 submit` from launching.
1650 One usage scenario is to run unit tests in the hook.
1652 The `p4-prepare-changelist` hook is executed right after preparing the default
1653 changelist message and before the editor is started. It takes one parameter,
1654 the name of the file that contains the changelist text. Exiting with a non-zero
1655 status from the script will abort the process.
1657 The purpose of the hook is to edit the message file in place, and it is not
1658 supressed by the `--no-verify` option. This hook is called even if
1659 `--prepare-p4-only` is set.
1661 The `p4-changelist` hook is executed after the changelist message has been
1662 edited by the user. It can be bypassed with the `--no-verify` option. It
1663 takes a single parameter, the name of the file that holds the proposed
1664 changelist text. Exiting with a non-zero status causes the command to abort.
1666 The hook is allowed to edit the changelist file and can be used to normalize
1667 the text into some project standard format. It can also be used to refuse the
1668 Submit after inspect the message file.
1670 The `p4-post-changelist` hook is invoked after the submit has successfully
1671 occured in P4. It takes no parameters and is meant primarily for notification
1672 and cannot affect the outcome of the git p4 submit action.
1675 self.usage += " [name of git branch to submit into perforce depot]"
1676 self.origin = ""
1677 self.detectRenames = False
1678 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1679 self.dry_run = False
1680 self.shelve = False
1681 self.update_shelve = list()
1682 self.commit = ""
1683 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1684 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1685 self.prepare_p4_only = False
1686 self.conflict_behavior = None
1687 self.isWindows = (platform.system() == "Windows")
1688 self.exportLabels = False
1689 self.p4HasMoveCommand = p4_has_move_command()
1690 self.branch = None
1691 self.no_verify = False
1693 if gitConfig('git-p4.largeFileSystem'):
1694 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1696 def check(self):
1697 if len(p4CmdList("opened ...")) > 0:
1698 die("You have files opened with perforce! Close them before starting the sync.")
1700 def separate_jobs_from_description(self, message):
1701 """Extract and return a possible Jobs field in the commit
1702 message. It goes into a separate section in the p4 change
1703 specification.
1705 A jobs line starts with "Jobs:" and looks like a new field
1706 in a form. Values are white-space separated on the same
1707 line or on following lines that start with a tab.
1709 This does not parse and extract the full git commit message
1710 like a p4 form. It just sees the Jobs: line as a marker
1711 to pass everything from then on directly into the p4 form,
1712 but outside the description section.
1714 Return a tuple (stripped log message, jobs string)."""
1716 m = re.search(r'^Jobs:', message, re.MULTILINE)
1717 if m is None:
1718 return (message, None)
1720 jobtext = message[m.start():]
1721 stripped_message = message[:m.start()].rstrip()
1722 return (stripped_message, jobtext)
1724 def prepareLogMessage(self, template, message, jobs):
1725 """Edits the template returned from "p4 change -o" to insert
1726 the message in the Description field, and the jobs text in
1727 the Jobs field."""
1728 result = ""
1730 inDescriptionSection = False
1732 for line in template.split("\n"):
1733 if line.startswith("#"):
1734 result += line + "\n"
1735 continue
1737 if inDescriptionSection:
1738 if line.startswith("Files:") or line.startswith("Jobs:"):
1739 inDescriptionSection = False
1740 # insert Jobs section
1741 if jobs:
1742 result += jobs + "\n"
1743 else:
1744 continue
1745 else:
1746 if line.startswith("Description:"):
1747 inDescriptionSection = True
1748 line += "\n"
1749 for messageLine in message.split("\n"):
1750 line += "\t" + messageLine + "\n"
1752 result += line + "\n"
1754 return result
1756 def patchRCSKeywords(self, file, pattern):
1757 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1758 (handle, outFileName) = tempfile.mkstemp(dir='.')
1759 try:
1760 outFile = os.fdopen(handle, "w+")
1761 inFile = open(file, "r")
1762 regexp = re.compile(pattern, re.VERBOSE)
1763 for line in inFile.readlines():
1764 line = regexp.sub(r'$\1$', line)
1765 outFile.write(line)
1766 inFile.close()
1767 outFile.close()
1768 # Forcibly overwrite the original file
1769 os.unlink(file)
1770 shutil.move(outFileName, file)
1771 except:
1772 # cleanup our temporary file
1773 os.unlink(outFileName)
1774 print("Failed to strip RCS keywords in %s" % file)
1775 raise
1777 print("Patched up RCS keywords in %s" % file)
1779 def p4UserForCommit(self,id):
1780 # Return the tuple (perforce user,git email) for a given git commit id
1781 self.getUserMapFromPerforceServer()
1782 gitEmail = read_pipe(["git", "log", "--max-count=1",
1783 "--format=%ae", id])
1784 gitEmail = gitEmail.strip()
1785 if gitEmail not in self.emails:
1786 return (None,gitEmail)
1787 else:
1788 return (self.emails[gitEmail],gitEmail)
1790 def checkValidP4Users(self,commits):
1791 # check if any git authors cannot be mapped to p4 users
1792 for id in commits:
1793 (user,email) = self.p4UserForCommit(id)
1794 if not user:
1795 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1796 if gitConfigBool("git-p4.allowMissingP4Users"):
1797 print("%s" % msg)
1798 else:
1799 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1801 def lastP4Changelist(self):
1802 # Get back the last changelist number submitted in this client spec. This
1803 # then gets used to patch up the username in the change. If the same
1804 # client spec is being used by multiple processes then this might go
1805 # wrong.
1806 results = p4CmdList("client -o") # find the current client
1807 client = None
1808 for r in results:
1809 if 'Client' in r:
1810 client = r['Client']
1811 break
1812 if not client:
1813 die("could not get client spec")
1814 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1815 for r in results:
1816 if 'change' in r:
1817 return r['change']
1818 die("Could not get changelist number for last submit - cannot patch up user details")
1820 def modifyChangelistUser(self, changelist, newUser):
1821 # fixup the user field of a changelist after it has been submitted.
1822 changes = p4CmdList("change -o %s" % changelist)
1823 if len(changes) != 1:
1824 die("Bad output from p4 change modifying %s to user %s" %
1825 (changelist, newUser))
1827 c = changes[0]
1828 if c['User'] == newUser: return # nothing to do
1829 c['User'] = newUser
1830 # p4 does not understand format version 3 and above
1831 input = marshal.dumps(c, 2)
1833 result = p4CmdList("change -f -i", stdin=input)
1834 for r in result:
1835 if 'code' in r:
1836 if r['code'] == 'error':
1837 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1838 if 'data' in r:
1839 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1840 return
1841 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1843 def canChangeChangelists(self):
1844 # check to see if we have p4 admin or super-user permissions, either of
1845 # which are required to modify changelists.
1846 results = p4CmdList(["protects", self.depotPath])
1847 for r in results:
1848 if 'perm' in r:
1849 if r['perm'] == 'admin':
1850 return 1
1851 if r['perm'] == 'super':
1852 return 1
1853 return 0
1855 def prepareSubmitTemplate(self, changelist=None):
1856 """Run "p4 change -o" to grab a change specification template.
1857 This does not use "p4 -G", as it is nice to keep the submission
1858 template in original order, since a human might edit it.
1860 Remove lines in the Files section that show changes to files
1861 outside the depot path we're committing into."""
1863 [upstream, settings] = findUpstreamBranchPoint()
1865 template = """\
1866 # A Perforce Change Specification.
1868 # Change: The change number. 'new' on a new changelist.
1869 # Date: The date this specification was last modified.
1870 # Client: The client on which the changelist was created. Read-only.
1871 # User: The user who created the changelist.
1872 # Status: Either 'pending' or 'submitted'. Read-only.
1873 # Type: Either 'public' or 'restricted'. Default is 'public'.
1874 # Description: Comments about the changelist. Required.
1875 # Jobs: What opened jobs are to be closed by this changelist.
1876 # You may delete jobs from this list. (New changelists only.)
1877 # Files: What opened files from the default changelist are to be added
1878 # to this changelist. You may delete files from this list.
1879 # (New changelists only.)
1881 files_list = []
1882 inFilesSection = False
1883 change_entry = None
1884 args = ['change', '-o']
1885 if changelist:
1886 args.append(str(changelist))
1887 for entry in p4CmdList(args):
1888 if 'code' not in entry:
1889 continue
1890 if entry['code'] == 'stat':
1891 change_entry = entry
1892 break
1893 if not change_entry:
1894 die('Failed to decode output of p4 change -o')
1895 for key, value in change_entry.items():
1896 if key.startswith('File'):
1897 if 'depot-paths' in settings:
1898 if not [p for p in settings['depot-paths']
1899 if p4PathStartsWith(value, p)]:
1900 continue
1901 else:
1902 if not p4PathStartsWith(value, self.depotPath):
1903 continue
1904 files_list.append(value)
1905 continue
1906 # Output in the order expected by prepareLogMessage
1907 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1908 if key not in change_entry:
1909 continue
1910 template += '\n'
1911 template += key + ':'
1912 if key == 'Description':
1913 template += '\n'
1914 for field_line in change_entry[key].splitlines():
1915 template += '\t'+field_line+'\n'
1916 if len(files_list) > 0:
1917 template += '\n'
1918 template += 'Files:\n'
1919 for path in files_list:
1920 template += '\t'+path+'\n'
1921 return template
1923 def edit_template(self, template_file):
1924 """Invoke the editor to let the user change the submission
1925 message. Return true if okay to continue with the submit."""
1927 # if configured to skip the editing part, just submit
1928 if gitConfigBool("git-p4.skipSubmitEdit"):
1929 return True
1931 # look at the modification time, to check later if the user saved
1932 # the file
1933 mtime = os.stat(template_file).st_mtime
1935 # invoke the editor
1936 if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
1937 editor = os.environ.get("P4EDITOR")
1938 else:
1939 editor = read_pipe("git var GIT_EDITOR").strip()
1940 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1942 # If the file was not saved, prompt to see if this patch should
1943 # be skipped. But skip this verification step if configured so.
1944 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1945 return True
1947 # modification time updated means user saved the file
1948 if os.stat(template_file).st_mtime > mtime:
1949 return True
1951 response = prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1952 if response == 'y':
1953 return True
1954 if response == 'n':
1955 return False
1957 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1958 # diff
1959 if "P4DIFF" in os.environ:
1960 del(os.environ["P4DIFF"])
1961 diff = ""
1962 for editedFile in editedFiles:
1963 diff += p4_read_pipe(['diff', '-du',
1964 wildcard_encode(editedFile)])
1966 # new file diff
1967 newdiff = ""
1968 for newFile in filesToAdd:
1969 newdiff += "==== new file ====\n"
1970 newdiff += "--- /dev/null\n"
1971 newdiff += "+++ %s\n" % newFile
1973 is_link = os.path.islink(newFile)
1974 expect_link = newFile in symlinks
1976 if is_link and expect_link:
1977 newdiff += "+%s\n" % os.readlink(newFile)
1978 else:
1979 f = open(newFile, "r")
1980 for line in f.readlines():
1981 newdiff += "+" + line
1982 f.close()
1984 return (diff + newdiff).replace('\r\n', '\n')
1986 def applyCommit(self, id):
1987 """Apply one commit, return True if it succeeded."""
1989 print("Applying", read_pipe(["git", "show", "-s",
1990 "--format=format:%h %s", id]))
1992 (p4User, gitEmail) = self.p4UserForCommit(id)
1994 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1995 filesToAdd = set()
1996 filesToChangeType = set()
1997 filesToDelete = set()
1998 editedFiles = set()
1999 pureRenameCopy = set()
2000 symlinks = set()
2001 filesToChangeExecBit = {}
2002 all_files = list()
2004 for line in diff:
2005 diff = parseDiffTreeEntry(line)
2006 modifier = diff['status']
2007 path = diff['src']
2008 all_files.append(path)
2010 if modifier == "M":
2011 p4_edit(path)
2012 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2013 filesToChangeExecBit[path] = diff['dst_mode']
2014 editedFiles.add(path)
2015 elif modifier == "A":
2016 filesToAdd.add(path)
2017 filesToChangeExecBit[path] = diff['dst_mode']
2018 if path in filesToDelete:
2019 filesToDelete.remove(path)
2021 dst_mode = int(diff['dst_mode'], 8)
2022 if dst_mode == 0o120000:
2023 symlinks.add(path)
2025 elif modifier == "D":
2026 filesToDelete.add(path)
2027 if path in filesToAdd:
2028 filesToAdd.remove(path)
2029 elif modifier == "C":
2030 src, dest = diff['src'], diff['dst']
2031 all_files.append(dest)
2032 p4_integrate(src, dest)
2033 pureRenameCopy.add(dest)
2034 if diff['src_sha1'] != diff['dst_sha1']:
2035 p4_edit(dest)
2036 pureRenameCopy.discard(dest)
2037 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2038 p4_edit(dest)
2039 pureRenameCopy.discard(dest)
2040 filesToChangeExecBit[dest] = diff['dst_mode']
2041 if self.isWindows:
2042 # turn off read-only attribute
2043 os.chmod(dest, stat.S_IWRITE)
2044 os.unlink(dest)
2045 editedFiles.add(dest)
2046 elif modifier == "R":
2047 src, dest = diff['src'], diff['dst']
2048 all_files.append(dest)
2049 if self.p4HasMoveCommand:
2050 p4_edit(src) # src must be open before move
2051 p4_move(src, dest) # opens for (move/delete, move/add)
2052 else:
2053 p4_integrate(src, dest)
2054 if diff['src_sha1'] != diff['dst_sha1']:
2055 p4_edit(dest)
2056 else:
2057 pureRenameCopy.add(dest)
2058 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
2059 if not self.p4HasMoveCommand:
2060 p4_edit(dest) # with move: already open, writable
2061 filesToChangeExecBit[dest] = diff['dst_mode']
2062 if not self.p4HasMoveCommand:
2063 if self.isWindows:
2064 os.chmod(dest, stat.S_IWRITE)
2065 os.unlink(dest)
2066 filesToDelete.add(src)
2067 editedFiles.add(dest)
2068 elif modifier == "T":
2069 filesToChangeType.add(path)
2070 else:
2071 die("unknown modifier %s for %s" % (modifier, path))
2073 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
2074 patchcmd = diffcmd + " | git apply "
2075 tryPatchCmd = patchcmd + "--check -"
2076 applyPatchCmd = patchcmd + "--check --apply -"
2077 patch_succeeded = True
2079 if verbose:
2080 print("TryPatch: %s" % tryPatchCmd)
2082 if os.system(tryPatchCmd) != 0:
2083 fixed_rcs_keywords = False
2084 patch_succeeded = False
2085 print("Unfortunately applying the change failed!")
2087 # Patch failed, maybe it's just RCS keyword woes. Look through
2088 # the patch to see if that's possible.
2089 if gitConfigBool("git-p4.attemptRCSCleanup"):
2090 file = None
2091 pattern = None
2092 kwfiles = {}
2093 for file in editedFiles | filesToDelete:
2094 # did this file's delta contain RCS keywords?
2095 pattern = p4_keywords_regexp_for_file(file)
2097 if pattern:
2098 # this file is a possibility...look for RCS keywords.
2099 regexp = re.compile(pattern, re.VERBOSE)
2100 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
2101 if regexp.search(line):
2102 if verbose:
2103 print("got keyword match on %s in %s in %s" % (pattern, line, file))
2104 kwfiles[file] = pattern
2105 break
2107 for file in kwfiles:
2108 if verbose:
2109 print("zapping %s with %s" % (line,pattern))
2110 # File is being deleted, so not open in p4. Must
2111 # disable the read-only bit on windows.
2112 if self.isWindows and file not in editedFiles:
2113 os.chmod(file, stat.S_IWRITE)
2114 self.patchRCSKeywords(file, kwfiles[file])
2115 fixed_rcs_keywords = True
2117 if fixed_rcs_keywords:
2118 print("Retrying the patch with RCS keywords cleaned up")
2119 if os.system(tryPatchCmd) == 0:
2120 patch_succeeded = True
2121 print("Patch succeesed this time with RCS keywords cleaned")
2123 if not patch_succeeded:
2124 for f in editedFiles:
2125 p4_revert(f)
2126 return False
2129 # Apply the patch for real, and do add/delete/+x handling.
2131 system(applyPatchCmd)
2133 for f in filesToChangeType:
2134 p4_edit(f, "-t", "auto")
2135 for f in filesToAdd:
2136 p4_add(f)
2137 for f in filesToDelete:
2138 p4_revert(f)
2139 p4_delete(f)
2141 # Set/clear executable bits
2142 for f in filesToChangeExecBit.keys():
2143 mode = filesToChangeExecBit[f]
2144 setP4ExecBit(f, mode)
2146 update_shelve = 0
2147 if len(self.update_shelve) > 0:
2148 update_shelve = self.update_shelve.pop(0)
2149 p4_reopen_in_change(update_shelve, all_files)
2152 # Build p4 change description, starting with the contents
2153 # of the git commit message.
2155 logMessage = extractLogMessageFromGitCommit(id)
2156 logMessage = logMessage.strip()
2157 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
2159 template = self.prepareSubmitTemplate(update_shelve)
2160 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
2162 if self.preserveUser:
2163 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
2165 if self.checkAuthorship and not self.p4UserIsMe(p4User):
2166 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
2167 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
2168 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2170 separatorLine = "######## everything below this line is just the diff #######\n"
2171 if not self.prepare_p4_only:
2172 submitTemplate += separatorLine
2173 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
2175 (handle, fileName) = tempfile.mkstemp()
2176 tmpFile = os.fdopen(handle, "w+b")
2177 if self.isWindows:
2178 submitTemplate = submitTemplate.replace("\n", "\r\n")
2179 tmpFile.write(encode_text_stream(submitTemplate))
2180 tmpFile.close()
2182 submitted = False
2184 try:
2185 # Allow the hook to edit the changelist text before presenting it
2186 # to the user.
2187 if not run_git_hook("p4-prepare-changelist", [fileName]):
2188 return False
2190 if self.prepare_p4_only:
2192 # Leave the p4 tree prepared, and the submit template around
2193 # and let the user decide what to do next
2195 submitted = True
2196 print("")
2197 print("P4 workspace prepared for submission.")
2198 print("To submit or revert, go to client workspace")
2199 print(" " + self.clientPath)
2200 print("")
2201 print("To submit, use \"p4 submit\" to write a new description,")
2202 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2203 " \"git p4\"." % fileName)
2204 print("You can delete the file \"%s\" when finished." % fileName)
2206 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2207 print("To preserve change ownership by user %s, you must\n" \
2208 "do \"p4 change -f <change>\" after submitting and\n" \
2209 "edit the User field.")
2210 if pureRenameCopy:
2211 print("After submitting, renamed files must be re-synced.")
2212 print("Invoke \"p4 sync -f\" on each of these files:")
2213 for f in pureRenameCopy:
2214 print(" " + f)
2216 print("")
2217 print("To revert the changes, use \"p4 revert ...\", and delete")
2218 print("the submit template file \"%s\"" % fileName)
2219 if filesToAdd:
2220 print("Since the commit adds new files, they must be deleted:")
2221 for f in filesToAdd:
2222 print(" " + f)
2223 print("")
2224 sys.stdout.flush()
2225 return True
2227 if self.edit_template(fileName):
2228 if not self.no_verify:
2229 if not run_git_hook("p4-changelist", [fileName]):
2230 print("The p4-changelist hook failed.")
2231 sys.stdout.flush()
2232 return False
2234 # read the edited message and submit
2235 tmpFile = open(fileName, "rb")
2236 message = decode_text_stream(tmpFile.read())
2237 tmpFile.close()
2238 if self.isWindows:
2239 message = message.replace("\r\n", "\n")
2240 if message.find(separatorLine) != -1:
2241 submitTemplate = message[:message.index(separatorLine)]
2242 else:
2243 submitTemplate = message
2245 if len(submitTemplate.strip()) == 0:
2246 print("Changelist is empty, aborting this changelist.")
2247 sys.stdout.flush()
2248 return False
2250 if update_shelve:
2251 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2252 elif self.shelve:
2253 p4_write_pipe(['shelve', '-i'], submitTemplate)
2254 else:
2255 p4_write_pipe(['submit', '-i'], submitTemplate)
2256 # The rename/copy happened by applying a patch that created a
2257 # new file. This leaves it writable, which confuses p4.
2258 for f in pureRenameCopy:
2259 p4_sync(f, "-f")
2261 if self.preserveUser:
2262 if p4User:
2263 # Get last changelist number. Cannot easily get it from
2264 # the submit command output as the output is
2265 # unmarshalled.
2266 changelist = self.lastP4Changelist()
2267 self.modifyChangelistUser(changelist, p4User)
2269 submitted = True
2271 run_git_hook("p4-post-changelist")
2272 finally:
2273 # Revert changes if we skip this patch
2274 if not submitted or self.shelve:
2275 if self.shelve:
2276 print ("Reverting shelved files.")
2277 else:
2278 print ("Submission cancelled, undoing p4 changes.")
2279 sys.stdout.flush()
2280 for f in editedFiles | filesToDelete:
2281 p4_revert(f)
2282 for f in filesToAdd:
2283 p4_revert(f)
2284 os.remove(f)
2286 if not self.prepare_p4_only:
2287 os.remove(fileName)
2288 return submitted
2290 # Export git tags as p4 labels. Create a p4 label and then tag
2291 # with that.
2292 def exportGitTags(self, gitTags):
2293 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2294 if len(validLabelRegexp) == 0:
2295 validLabelRegexp = defaultLabelRegexp
2296 m = re.compile(validLabelRegexp)
2298 for name in gitTags:
2300 if not m.match(name):
2301 if verbose:
2302 print("tag %s does not match regexp %s" % (name, validLabelRegexp))
2303 continue
2305 # Get the p4 commit this corresponds to
2306 logMessage = extractLogMessageFromGitCommit(name)
2307 values = extractSettingsGitLog(logMessage)
2309 if 'change' not in values:
2310 # a tag pointing to something not sent to p4; ignore
2311 if verbose:
2312 print("git tag %s does not give a p4 commit" % name)
2313 continue
2314 else:
2315 changelist = values['change']
2317 # Get the tag details.
2318 inHeader = True
2319 isAnnotated = False
2320 body = []
2321 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2322 l = l.strip()
2323 if inHeader:
2324 if re.match(r'tag\s+', l):
2325 isAnnotated = True
2326 elif re.match(r'\s*$', l):
2327 inHeader = False
2328 continue
2329 else:
2330 body.append(l)
2332 if not isAnnotated:
2333 body = ["lightweight tag imported by git p4\n"]
2335 # Create the label - use the same view as the client spec we are using
2336 clientSpec = getClientSpec()
2338 labelTemplate = "Label: %s\n" % name
2339 labelTemplate += "Description:\n"
2340 for b in body:
2341 labelTemplate += "\t" + b + "\n"
2342 labelTemplate += "View:\n"
2343 for depot_side in clientSpec.mappings:
2344 labelTemplate += "\t%s\n" % depot_side
2346 if self.dry_run:
2347 print("Would create p4 label %s for tag" % name)
2348 elif self.prepare_p4_only:
2349 print("Not creating p4 label %s for tag due to option" \
2350 " --prepare-p4-only" % name)
2351 else:
2352 p4_write_pipe(["label", "-i"], labelTemplate)
2354 # Use the label
2355 p4_system(["tag", "-l", name] +
2356 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2358 if verbose:
2359 print("created p4 label for tag %s" % name)
2361 def run(self, args):
2362 if len(args) == 0:
2363 self.master = currentGitBranch()
2364 elif len(args) == 1:
2365 self.master = args[0]
2366 if not branchExists(self.master):
2367 die("Branch %s does not exist" % self.master)
2368 else:
2369 return False
2371 for i in self.update_shelve:
2372 if i <= 0:
2373 sys.exit("invalid changelist %d" % i)
2375 if self.master:
2376 allowSubmit = gitConfig("git-p4.allowSubmit")
2377 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2378 die("%s is not in git-p4.allowSubmit" % self.master)
2380 [upstream, settings] = findUpstreamBranchPoint()
2381 self.depotPath = settings['depot-paths'][0]
2382 if len(self.origin) == 0:
2383 self.origin = upstream
2385 if len(self.update_shelve) > 0:
2386 self.shelve = True
2388 if self.preserveUser:
2389 if not self.canChangeChangelists():
2390 die("Cannot preserve user names without p4 super-user or admin permissions")
2392 # if not set from the command line, try the config file
2393 if self.conflict_behavior is None:
2394 val = gitConfig("git-p4.conflict")
2395 if val:
2396 if val not in self.conflict_behavior_choices:
2397 die("Invalid value '%s' for config git-p4.conflict" % val)
2398 else:
2399 val = "ask"
2400 self.conflict_behavior = val
2402 if self.verbose:
2403 print("Origin branch is " + self.origin)
2405 if len(self.depotPath) == 0:
2406 print("Internal error: cannot locate perforce depot path from existing branches")
2407 sys.exit(128)
2409 self.useClientSpec = False
2410 if gitConfigBool("git-p4.useclientspec"):
2411 self.useClientSpec = True
2412 if self.useClientSpec:
2413 self.clientSpecDirs = getClientSpec()
2415 # Check for the existence of P4 branches
2416 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2418 if self.useClientSpec and not branchesDetected:
2419 # all files are relative to the client spec
2420 self.clientPath = getClientRoot()
2421 else:
2422 self.clientPath = p4Where(self.depotPath)
2424 if self.clientPath == "":
2425 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2427 print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
2428 self.oldWorkingDirectory = os.getcwd()
2430 # ensure the clientPath exists
2431 new_client_dir = False
2432 if not os.path.exists(self.clientPath):
2433 new_client_dir = True
2434 os.makedirs(self.clientPath)
2436 chdir(self.clientPath, is_client_path=True)
2437 if self.dry_run:
2438 print("Would synchronize p4 checkout in %s" % self.clientPath)
2439 else:
2440 print("Synchronizing p4 checkout...")
2441 if new_client_dir:
2442 # old one was destroyed, and maybe nobody told p4
2443 p4_sync("...", "-f")
2444 else:
2445 p4_sync("...")
2446 self.check()
2448 commits = []
2449 if self.master:
2450 committish = self.master
2451 else:
2452 committish = 'HEAD'
2454 if self.commit != "":
2455 if self.commit.find("..") != -1:
2456 limits_ish = self.commit.split("..")
2457 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2458 commits.append(line.strip())
2459 commits.reverse()
2460 else:
2461 commits.append(self.commit)
2462 else:
2463 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2464 commits.append(line.strip())
2465 commits.reverse()
2467 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2468 self.checkAuthorship = False
2469 else:
2470 self.checkAuthorship = True
2472 if self.preserveUser:
2473 self.checkValidP4Users(commits)
2476 # Build up a set of options to be passed to diff when
2477 # submitting each commit to p4.
2479 if self.detectRenames:
2480 # command-line -M arg
2481 self.diffOpts = "-M"
2482 else:
2483 # If not explicitly set check the config variable
2484 detectRenames = gitConfig("git-p4.detectRenames")
2486 if detectRenames.lower() == "false" or detectRenames == "":
2487 self.diffOpts = ""
2488 elif detectRenames.lower() == "true":
2489 self.diffOpts = "-M"
2490 else:
2491 self.diffOpts = "-M%s" % detectRenames
2493 # no command-line arg for -C or --find-copies-harder, just
2494 # config variables
2495 detectCopies = gitConfig("git-p4.detectCopies")
2496 if detectCopies.lower() == "false" or detectCopies == "":
2497 pass
2498 elif detectCopies.lower() == "true":
2499 self.diffOpts += " -C"
2500 else:
2501 self.diffOpts += " -C%s" % detectCopies
2503 if gitConfigBool("git-p4.detectCopiesHarder"):
2504 self.diffOpts += " --find-copies-harder"
2506 num_shelves = len(self.update_shelve)
2507 if num_shelves > 0 and num_shelves != len(commits):
2508 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2509 (len(commits), num_shelves))
2511 if not self.no_verify:
2512 try:
2513 if not run_git_hook("p4-pre-submit"):
2514 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " \
2515 "this pre-submission check by adding\nthe command line option '--no-verify', " \
2516 "however,\nthis will also skip the p4-changelist hook as well.")
2517 sys.exit(1)
2518 except Exception as e:
2519 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "\
2520 "with the error '{0}'".format(e.message) )
2521 sys.exit(1)
2524 # Apply the commits, one at a time. On failure, ask if should
2525 # continue to try the rest of the patches, or quit.
2527 if self.dry_run:
2528 print("Would apply")
2529 applied = []
2530 last = len(commits) - 1
2531 for i, commit in enumerate(commits):
2532 if self.dry_run:
2533 print(" ", read_pipe(["git", "show", "-s",
2534 "--format=format:%h %s", commit]))
2535 ok = True
2536 else:
2537 ok = self.applyCommit(commit)
2538 if ok:
2539 applied.append(commit)
2540 else:
2541 if self.prepare_p4_only and i < last:
2542 print("Processing only the first commit due to option" \
2543 " --prepare-p4-only")
2544 break
2545 if i < last:
2546 # prompt for what to do, or use the option/variable
2547 if self.conflict_behavior == "ask":
2548 print("What do you want to do?")
2549 response = prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2550 elif self.conflict_behavior == "skip":
2551 response = "s"
2552 elif self.conflict_behavior == "quit":
2553 response = "q"
2554 else:
2555 die("Unknown conflict_behavior '%s'" %
2556 self.conflict_behavior)
2558 if response == "s":
2559 print("Skipping this commit, but applying the rest")
2560 if response == "q":
2561 print("Quitting")
2562 break
2564 chdir(self.oldWorkingDirectory)
2565 shelved_applied = "shelved" if self.shelve else "applied"
2566 if self.dry_run:
2567 pass
2568 elif self.prepare_p4_only:
2569 pass
2570 elif len(commits) == len(applied):
2571 print("All commits {0}!".format(shelved_applied))
2573 sync = P4Sync()
2574 if self.branch:
2575 sync.branch = self.branch
2576 if self.disable_p4sync:
2577 sync.sync_origin_only()
2578 else:
2579 sync.run([])
2581 if not self.disable_rebase:
2582 rebase = P4Rebase()
2583 rebase.rebase()
2585 else:
2586 if len(applied) == 0:
2587 print("No commits {0}.".format(shelved_applied))
2588 else:
2589 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2590 for c in commits:
2591 if c in applied:
2592 star = "*"
2593 else:
2594 star = " "
2595 print(star, read_pipe(["git", "show", "-s",
2596 "--format=format:%h %s", c]))
2597 print("You will have to do 'git p4 sync' and rebase.")
2599 if gitConfigBool("git-p4.exportLabels"):
2600 self.exportLabels = True
2602 if self.exportLabels:
2603 p4Labels = getP4Labels(self.depotPath)
2604 gitTags = getGitTags()
2606 missingGitTags = gitTags - p4Labels
2607 self.exportGitTags(missingGitTags)
2609 # exit with error unless everything applied perfectly
2610 if len(commits) != len(applied):
2611 sys.exit(1)
2613 return True
2615 class View(object):
2616 """Represent a p4 view ("p4 help views"), and map files in a
2617 repo according to the view."""
2619 def __init__(self, client_name):
2620 self.mappings = []
2621 self.client_prefix = "//%s/" % client_name
2622 # cache results of "p4 where" to lookup client file locations
2623 self.client_spec_path_cache = {}
2625 def append(self, view_line):
2626 """Parse a view line, splitting it into depot and client
2627 sides. Append to self.mappings, preserving order. This
2628 is only needed for tag creation."""
2630 # Split the view line into exactly two words. P4 enforces
2631 # structure on these lines that simplifies this quite a bit.
2633 # Either or both words may be double-quoted.
2634 # Single quotes do not matter.
2635 # Double-quote marks cannot occur inside the words.
2636 # A + or - prefix is also inside the quotes.
2637 # There are no quotes unless they contain a space.
2638 # The line is already white-space stripped.
2639 # The two words are separated by a single space.
2641 if view_line[0] == '"':
2642 # First word is double quoted. Find its end.
2643 close_quote_index = view_line.find('"', 1)
2644 if close_quote_index <= 0:
2645 die("No first-word closing quote found: %s" % view_line)
2646 depot_side = view_line[1:close_quote_index]
2647 # skip closing quote and space
2648 rhs_index = close_quote_index + 1 + 1
2649 else:
2650 space_index = view_line.find(" ")
2651 if space_index <= 0:
2652 die("No word-splitting space found: %s" % view_line)
2653 depot_side = view_line[0:space_index]
2654 rhs_index = space_index + 1
2656 # prefix + means overlay on previous mapping
2657 if depot_side.startswith("+"):
2658 depot_side = depot_side[1:]
2660 # prefix - means exclude this path, leave out of mappings
2661 exclude = False
2662 if depot_side.startswith("-"):
2663 exclude = True
2664 depot_side = depot_side[1:]
2666 if not exclude:
2667 self.mappings.append(depot_side)
2669 def convert_client_path(self, clientFile):
2670 # chop off //client/ part to make it relative
2671 if not decode_path(clientFile).startswith(self.client_prefix):
2672 die("No prefix '%s' on clientFile '%s'" %
2673 (self.client_prefix, clientFile))
2674 return clientFile[len(self.client_prefix):]
2676 def update_client_spec_path_cache(self, files):
2677 """ Caching file paths by "p4 where" batch query """
2679 # List depot file paths exclude that already cached
2680 fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]
2682 if len(fileArgs) == 0:
2683 return # All files in cache
2685 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2686 for res in where_result:
2687 if "code" in res and res["code"] == "error":
2688 # assume error is "... file(s) not in client view"
2689 continue
2690 if "clientFile" not in res:
2691 die("No clientFile in 'p4 where' output")
2692 if "unmap" in res:
2693 # it will list all of them, but only one not unmap-ped
2694 continue
2695 depot_path = decode_path(res['depotFile'])
2696 if gitConfigBool("core.ignorecase"):
2697 depot_path = depot_path.lower()
2698 self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])
2700 # not found files or unmap files set to ""
2701 for depotFile in fileArgs:
2702 depotFile = decode_path(depotFile)
2703 if gitConfigBool("core.ignorecase"):
2704 depotFile = depotFile.lower()
2705 if depotFile not in self.client_spec_path_cache:
2706 self.client_spec_path_cache[depotFile] = b''
2708 def map_in_client(self, depot_path):
2709 """Return the relative location in the client where this
2710 depot file should live. Returns "" if the file should
2711 not be mapped in the client."""
2713 if gitConfigBool("core.ignorecase"):
2714 depot_path = depot_path.lower()
2716 if depot_path in self.client_spec_path_cache:
2717 return self.client_spec_path_cache[depot_path]
2719 die( "Error: %s is not found in client spec path" % depot_path )
2720 return ""
2722 def cloneExcludeCallback(option, opt_str, value, parser):
2723 # prepend "/" because the first "/" was consumed as part of the option itself.
2724 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2725 parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2727 class P4Sync(Command, P4UserMap):
2729 def __init__(self):
2730 Command.__init__(self)
2731 P4UserMap.__init__(self)
2732 self.options = [
2733 optparse.make_option("--branch", dest="branch"),
2734 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2735 optparse.make_option("--changesfile", dest="changesFile"),
2736 optparse.make_option("--silent", dest="silent", action="store_true"),
2737 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2738 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2739 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2740 help="Import into refs/heads/ , not refs/remotes"),
2741 optparse.make_option("--max-changes", dest="maxChanges",
2742 help="Maximum number of changes to import"),
2743 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2744 help="Internal block size to use when iteratively calling p4 changes"),
2745 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2746 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2747 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2748 help="Only sync files that are included in the Perforce Client Spec"),
2749 optparse.make_option("-/", dest="cloneExclude",
2750 action="callback", callback=cloneExcludeCallback, type="string",
2751 help="exclude depot path"),
2753 self.description = """Imports from Perforce into a git repository.\n
2754 example:
2755 //depot/my/project/ -- to import the current head
2756 //depot/my/project/@all -- to import everything
2757 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2759 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2761 self.usage += " //depot/path[@revRange]"
2762 self.silent = False
2763 self.createdBranches = set()
2764 self.committedChanges = set()
2765 self.branch = ""
2766 self.detectBranches = False
2767 self.detectLabels = False
2768 self.importLabels = False
2769 self.changesFile = ""
2770 self.syncWithOrigin = True
2771 self.importIntoRemotes = True
2772 self.maxChanges = ""
2773 self.changes_block_size = None
2774 self.keepRepoPath = False
2775 self.depotPaths = None
2776 self.p4BranchesInGit = []
2777 self.cloneExclude = []
2778 self.useClientSpec = False
2779 self.useClientSpec_from_options = False
2780 self.clientSpecDirs = None
2781 self.tempBranches = []
2782 self.tempBranchLocation = "refs/git-p4-tmp"
2783 self.largeFileSystem = None
2784 self.suppress_meta_comment = False
2786 if gitConfig('git-p4.largeFileSystem'):
2787 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2788 self.largeFileSystem = largeFileSystemConstructor(
2789 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2792 if gitConfig("git-p4.syncFromOrigin") == "false":
2793 self.syncWithOrigin = False
2795 self.depotPaths = []
2796 self.changeRange = ""
2797 self.previousDepotPaths = []
2798 self.hasOrigin = False
2800 # map from branch depot path to parent branch
2801 self.knownBranches = {}
2802 self.initialParents = {}
2804 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2805 self.labels = {}
2807 # Force a checkpoint in fast-import and wait for it to finish
2808 def checkpoint(self):
2809 self.gitStream.write("checkpoint\n\n")
2810 self.gitStream.write("progress checkpoint\n\n")
2811 self.gitStream.flush()
2812 out = self.gitOutput.readline()
2813 if self.verbose:
2814 print("checkpoint finished: " + out)
2816 def isPathWanted(self, path):
2817 for p in self.cloneExclude:
2818 if p.endswith("/"):
2819 if p4PathStartsWith(path, p):
2820 return False
2821 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2822 elif path.lower() == p.lower():
2823 return False
2824 for p in self.depotPaths:
2825 if p4PathStartsWith(path, decode_path(p)):
2826 return True
2827 return False
2829 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
2830 files = []
2831 fnum = 0
2832 while "depotFile%s" % fnum in commit:
2833 path = commit["depotFile%s" % fnum]
2834 found = self.isPathWanted(decode_path(path))
2835 if not found:
2836 fnum = fnum + 1
2837 continue
2839 file = {}
2840 file["path"] = path
2841 file["rev"] = commit["rev%s" % fnum]
2842 file["action"] = commit["action%s" % fnum]
2843 file["type"] = commit["type%s" % fnum]
2844 if shelved:
2845 file["shelved_cl"] = int(shelved_cl)
2846 files.append(file)
2847 fnum = fnum + 1
2848 return files
2850 def extractJobsFromCommit(self, commit):
2851 jobs = []
2852 jnum = 0
2853 while "job%s" % jnum in commit:
2854 job = commit["job%s" % jnum]
2855 jobs.append(job)
2856 jnum = jnum + 1
2857 return jobs
2859 def stripRepoPath(self, path, prefixes):
2860 """When streaming files, this is called to map a p4 depot path
2861 to where it should go in git. The prefixes are either
2862 self.depotPaths, or self.branchPrefixes in the case of
2863 branch detection."""
2865 if self.useClientSpec:
2866 # branch detection moves files up a level (the branch name)
2867 # from what client spec interpretation gives
2868 path = decode_path(self.clientSpecDirs.map_in_client(path))
2869 if self.detectBranches:
2870 for b in self.knownBranches:
2871 if p4PathStartsWith(path, b + "/"):
2872 path = path[len(b)+1:]
2874 elif self.keepRepoPath:
2875 # Preserve everything in relative path name except leading
2876 # //depot/; just look at first prefix as they all should
2877 # be in the same depot.
2878 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2879 if p4PathStartsWith(path, depot):
2880 path = path[len(depot):]
2882 else:
2883 for p in prefixes:
2884 if p4PathStartsWith(path, p):
2885 path = path[len(p):]
2886 break
2888 path = wildcard_decode(path)
2889 return path
2891 def splitFilesIntoBranches(self, commit):
2892 """Look at each depotFile in the commit to figure out to what
2893 branch it belongs."""
2895 if self.clientSpecDirs:
2896 files = self.extractFilesFromCommit(commit)
2897 self.clientSpecDirs.update_client_spec_path_cache(files)
2899 branches = {}
2900 fnum = 0
2901 while "depotFile%s" % fnum in commit:
2902 raw_path = commit["depotFile%s" % fnum]
2903 path = decode_path(raw_path)
2904 found = self.isPathWanted(path)
2905 if not found:
2906 fnum = fnum + 1
2907 continue
2909 file = {}
2910 file["path"] = raw_path
2911 file["rev"] = commit["rev%s" % fnum]
2912 file["action"] = commit["action%s" % fnum]
2913 file["type"] = commit["type%s" % fnum]
2914 fnum = fnum + 1
2916 # start with the full relative path where this file would
2917 # go in a p4 client
2918 if self.useClientSpec:
2919 relPath = decode_path(self.clientSpecDirs.map_in_client(path))
2920 else:
2921 relPath = self.stripRepoPath(path, self.depotPaths)
2923 for branch in self.knownBranches.keys():
2924 # add a trailing slash so that a commit into qt/4.2foo
2925 # doesn't end up in qt/4.2, e.g.
2926 if p4PathStartsWith(relPath, branch + "/"):
2927 if branch not in branches:
2928 branches[branch] = []
2929 branches[branch].append(file)
2930 break
2932 return branches
2934 def writeToGitStream(self, gitMode, relPath, contents):
2935 self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath)))
2936 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2937 for d in contents:
2938 self.gitStream.write(d)
2939 self.gitStream.write('\n')
2941 def encodeWithUTF8(self, path):
2942 try:
2943 path.decode('ascii')
2944 except:
2945 encoding = 'utf8'
2946 if gitConfig('git-p4.pathEncoding'):
2947 encoding = gitConfig('git-p4.pathEncoding')
2948 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2949 if self.verbose:
2950 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
2951 return path
2953 # output one file from the P4 stream
2954 # - helper for streamP4Files
2956 def streamOneP4File(self, file, contents):
2957 file_path = file['depotFile']
2958 relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)
2960 if verbose:
2961 if 'fileSize' in self.stream_file:
2962 size = int(self.stream_file['fileSize'])
2963 else:
2964 size = 0 # deleted files don't get a fileSize apparently
2965 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file_path, relPath, size/1024/1024))
2966 sys.stdout.flush()
2968 (type_base, type_mods) = split_p4_type(file["type"])
2970 git_mode = "100644"
2971 if "x" in type_mods:
2972 git_mode = "100755"
2973 if type_base == "symlink":
2974 git_mode = "120000"
2975 # p4 print on a symlink sometimes contains "target\n";
2976 # if it does, remove the newline
2977 data = ''.join(decode_text_stream(c) for c in contents)
2978 if not data:
2979 # Some version of p4 allowed creating a symlink that pointed
2980 # to nothing. This causes p4 errors when checking out such
2981 # a change, and errors here too. Work around it by ignoring
2982 # the bad symlink; hopefully a future change fixes it.
2983 print("\nIgnoring empty symlink in %s" % file_path)
2984 return
2985 elif data[-1] == '\n':
2986 contents = [data[:-1]]
2987 else:
2988 contents = [data]
2990 if type_base == "utf16":
2991 # p4 delivers different text in the python output to -G
2992 # than it does when using "print -o", or normal p4 client
2993 # operations. utf16 is converted to ascii or utf8, perhaps.
2994 # But ascii text saved as -t utf16 is completely mangled.
2995 # Invoke print -o to get the real contents.
2997 # On windows, the newlines will always be mangled by print, so put
2998 # them back too. This is not needed to the cygwin windows version,
2999 # just the native "NT" type.
3001 try:
3002 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)
3003 except Exception as e:
3004 if 'Translation of file content failed' in str(e):
3005 type_base = 'binary'
3006 else:
3007 raise e
3008 else:
3009 if p4_version_string().find('/NT') >= 0:
3010 text = text.replace(b'\r\n', b'\n')
3011 contents = [ text ]
3013 if type_base == "apple":
3014 # Apple filetype files will be streamed as a concatenation of
3015 # its appledouble header and the contents. This is useless
3016 # on both macs and non-macs. If using "print -q -o xx", it
3017 # will create "xx" with the data, and "%xx" with the header.
3018 # This is also not very useful.
3020 # Ideally, someday, this script can learn how to generate
3021 # appledouble files directly and import those to git, but
3022 # non-mac machines can never find a use for apple filetype.
3023 print("\nIgnoring apple filetype file %s" % file['depotFile'])
3024 return
3026 # Note that we do not try to de-mangle keywords on utf16 files,
3027 # even though in theory somebody may want that.
3028 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
3029 if pattern:
3030 regexp = re.compile(pattern, re.VERBOSE)
3031 text = ''.join(decode_text_stream(c) for c in contents)
3032 text = regexp.sub(r'$\1$', text)
3033 contents = [ text ]
3035 if self.largeFileSystem:
3036 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
3038 self.writeToGitStream(git_mode, relPath, contents)
3040 def streamOneP4Deletion(self, file):
3041 relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)
3042 if verbose:
3043 sys.stdout.write("delete %s\n" % relPath)
3044 sys.stdout.flush()
3045 self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath)))
3047 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
3048 self.largeFileSystem.removeLargeFile(relPath)
3050 # handle another chunk of streaming data
3051 def streamP4FilesCb(self, marshalled):
3053 # catch p4 errors and complain
3054 err = None
3055 if "code" in marshalled:
3056 if marshalled["code"] == "error":
3057 if "data" in marshalled:
3058 err = marshalled["data"].rstrip()
3060 if not err and 'fileSize' in self.stream_file:
3061 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
3062 if required_bytes > 0:
3063 err = 'Not enough space left on %s! Free at least %i MB.' % (
3064 os.getcwd(), required_bytes/1024/1024
3067 if err:
3068 f = None
3069 if self.stream_have_file_info:
3070 if "depotFile" in self.stream_file:
3071 f = self.stream_file["depotFile"]
3072 # force a failure in fast-import, else an empty
3073 # commit will be made
3074 self.gitStream.write("\n")
3075 self.gitStream.write("die-now\n")
3076 self.gitStream.close()
3077 # ignore errors, but make sure it exits first
3078 self.importProcess.wait()
3079 if f:
3080 die("Error from p4 print for %s: %s" % (f, err))
3081 else:
3082 die("Error from p4 print: %s" % err)
3084 if 'depotFile' in marshalled and self.stream_have_file_info:
3085 # start of a new file - output the old one first
3086 self.streamOneP4File(self.stream_file, self.stream_contents)
3087 self.stream_file = {}
3088 self.stream_contents = []
3089 self.stream_have_file_info = False
3091 # pick up the new file information... for the
3092 # 'data' field we need to append to our array
3093 for k in marshalled.keys():
3094 if k == 'data':
3095 if 'streamContentSize' not in self.stream_file:
3096 self.stream_file['streamContentSize'] = 0
3097 self.stream_file['streamContentSize'] += len(marshalled['data'])
3098 self.stream_contents.append(marshalled['data'])
3099 else:
3100 self.stream_file[k] = marshalled[k]
3102 if (verbose and
3103 'streamContentSize' in self.stream_file and
3104 'fileSize' in self.stream_file and
3105 'depotFile' in self.stream_file):
3106 size = int(self.stream_file["fileSize"])
3107 if size > 0:
3108 progress = 100*self.stream_file['streamContentSize']/size
3109 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
3110 sys.stdout.flush()
3112 self.stream_have_file_info = True
3114 # Stream directly from "p4 files" into "git fast-import"
3115 def streamP4Files(self, files):
3116 filesForCommit = []
3117 filesToRead = []
3118 filesToDelete = []
3120 for f in files:
3121 filesForCommit.append(f)
3122 if f['action'] in self.delete_actions:
3123 filesToDelete.append(f)
3124 else:
3125 filesToRead.append(f)
3127 # deleted files...
3128 for f in filesToDelete:
3129 self.streamOneP4Deletion(f)
3131 if len(filesToRead) > 0:
3132 self.stream_file = {}
3133 self.stream_contents = []
3134 self.stream_have_file_info = False
3136 # curry self argument
3137 def streamP4FilesCbSelf(entry):
3138 self.streamP4FilesCb(entry)
3140 fileArgs = []
3141 for f in filesToRead:
3142 if 'shelved_cl' in f:
3143 # Handle shelved CLs using the "p4 print file@=N" syntax to print
3144 # the contents
3145 fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl']))
3146 else:
3147 fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev']))
3149 fileArgs.append(fileArg)
3151 p4CmdList(["-x", "-", "print"],
3152 stdin=fileArgs,
3153 cb=streamP4FilesCbSelf)
3155 # do the last chunk
3156 if 'depotFile' in self.stream_file:
3157 self.streamOneP4File(self.stream_file, self.stream_contents)
3159 def make_email(self, userid):
3160 if userid in self.users:
3161 return self.users[userid]
3162 else:
3163 return "%s <a@b>" % userid
3165 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
3166 """ Stream a p4 tag.
3167 commit is either a git commit, or a fast-import mark, ":<p4commit>"
3170 if verbose:
3171 print("writing tag %s for commit %s" % (labelName, commit))
3172 gitStream.write("tag %s\n" % labelName)
3173 gitStream.write("from %s\n" % commit)
3175 if 'Owner' in labelDetails:
3176 owner = labelDetails["Owner"]
3177 else:
3178 owner = None
3180 # Try to use the owner of the p4 label, or failing that,
3181 # the current p4 user id.
3182 if owner:
3183 email = self.make_email(owner)
3184 else:
3185 email = self.make_email(self.p4UserId())
3186 tagger = "%s %s %s" % (email, epoch, self.tz)
3188 gitStream.write("tagger %s\n" % tagger)
3190 print("labelDetails=",labelDetails)
3191 if 'Description' in labelDetails:
3192 description = labelDetails['Description']
3193 else:
3194 description = 'Label from git p4'
3196 gitStream.write("data %d\n" % len(description))
3197 gitStream.write(description)
3198 gitStream.write("\n")
3200 def inClientSpec(self, path):
3201 if not self.clientSpecDirs:
3202 return True
3203 inClientSpec = self.clientSpecDirs.map_in_client(path)
3204 if not inClientSpec and self.verbose:
3205 print('Ignoring file outside of client spec: {0}'.format(path))
3206 return inClientSpec
3208 def hasBranchPrefix(self, path):
3209 if not self.branchPrefixes:
3210 return True
3211 hasPrefix = [p for p in self.branchPrefixes
3212 if p4PathStartsWith(path, p)]
3213 if not hasPrefix and self.verbose:
3214 print('Ignoring file outside of prefix: {0}'.format(path))
3215 return hasPrefix
3217 def commit(self, details, files, branch, parent = "", allow_empty=False):
3218 epoch = details["time"]
3219 author = details["user"]
3220 jobs = self.extractJobsFromCommit(details)
3222 if self.verbose:
3223 print('commit into {0}'.format(branch))
3225 if self.clientSpecDirs:
3226 self.clientSpecDirs.update_client_spec_path_cache(files)
3228 files = [f for (f, path) in ((f, decode_path(f['path'])) for f in files)
3229 if self.inClientSpec(path) and self.hasBranchPrefix(path)]
3231 if gitConfigBool('git-p4.keepEmptyCommits'):
3232 allow_empty = True
3234 if not files and not allow_empty:
3235 print('Ignoring revision {0} as it would produce an empty commit.'
3236 .format(details['change']))
3237 return
3239 self.gitStream.write("commit %s\n" % branch)
3240 self.gitStream.write("mark :%s\n" % details["change"])
3241 self.committedChanges.add(int(details["change"]))
3242 committer = ""
3243 if author not in self.users:
3244 self.getUserMapFromPerforceServer()
3245 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3247 self.gitStream.write("committer %s\n" % committer)
3249 self.gitStream.write("data <<EOT\n")
3250 self.gitStream.write(details["desc"])
3251 if len(jobs) > 0:
3252 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3254 if not self.suppress_meta_comment:
3255 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3256 (','.join(self.branchPrefixes), details["change"]))
3257 if len(details['options']) > 0:
3258 self.gitStream.write(": options = %s" % details['options'])
3259 self.gitStream.write("]\n")
3261 self.gitStream.write("EOT\n\n")
3263 if len(parent) > 0:
3264 if self.verbose:
3265 print("parent %s" % parent)
3266 self.gitStream.write("from %s\n" % parent)
3268 self.streamP4Files(files)
3269 self.gitStream.write("\n")
3271 change = int(details["change"])
3273 if change in self.labels:
3274 label = self.labels[change]
3275 labelDetails = label[0]
3276 labelRevisions = label[1]
3277 if self.verbose:
3278 print("Change %s is labelled %s" % (change, labelDetails))
3280 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3281 for p in self.branchPrefixes])
3283 if len(files) == len(labelRevisions):
3285 cleanedFiles = {}
3286 for info in files:
3287 if info["action"] in self.delete_actions:
3288 continue
3289 cleanedFiles[info["depotFile"]] = info["rev"]
3291 if cleanedFiles == labelRevisions:
3292 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3294 else:
3295 if not self.silent:
3296 print("Tag %s does not match with change %s: files do not match."
3297 % (labelDetails["label"], change))
3299 else:
3300 if not self.silent:
3301 print("Tag %s does not match with change %s: file count is different."
3302 % (labelDetails["label"], change))
3304 # Build a dictionary of changelists and labels, for "detect-labels" option.
3305 def getLabels(self):
3306 self.labels = {}
3308 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3309 if len(l) > 0 and not self.silent:
3310 print("Finding files belonging to labels in %s" % self.depotPaths)
3312 for output in l:
3313 label = output["label"]
3314 revisions = {}
3315 newestChange = 0
3316 if self.verbose:
3317 print("Querying files for label %s" % label)
3318 for file in p4CmdList(["files"] +
3319 ["%s...@%s" % (p, label)
3320 for p in self.depotPaths]):
3321 revisions[file["depotFile"]] = file["rev"]
3322 change = int(file["change"])
3323 if change > newestChange:
3324 newestChange = change
3326 self.labels[newestChange] = [output, revisions]
3328 if self.verbose:
3329 print("Label changes: %s" % self.labels.keys())
3331 # Import p4 labels as git tags. A direct mapping does not
3332 # exist, so assume that if all the files are at the same revision
3333 # then we can use that, or it's something more complicated we should
3334 # just ignore.
3335 def importP4Labels(self, stream, p4Labels):
3336 if verbose:
3337 print("import p4 labels: " + ' '.join(p4Labels))
3339 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3340 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3341 if len(validLabelRegexp) == 0:
3342 validLabelRegexp = defaultLabelRegexp
3343 m = re.compile(validLabelRegexp)
3345 for name in p4Labels:
3346 commitFound = False
3348 if not m.match(name):
3349 if verbose:
3350 print("label %s does not match regexp %s" % (name,validLabelRegexp))
3351 continue
3353 if name in ignoredP4Labels:
3354 continue
3356 labelDetails = p4CmdList(['label', "-o", name])[0]
3358 # get the most recent changelist for each file in this label
3359 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3360 for p in self.depotPaths])
3362 if 'change' in change:
3363 # find the corresponding git commit; take the oldest commit
3364 changelist = int(change['change'])
3365 if changelist in self.committedChanges:
3366 gitCommit = ":%d" % changelist # use a fast-import mark
3367 commitFound = True
3368 else:
3369 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3370 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3371 if len(gitCommit) == 0:
3372 print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
3373 else:
3374 commitFound = True
3375 gitCommit = gitCommit.strip()
3377 if commitFound:
3378 # Convert from p4 time format
3379 try:
3380 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3381 except ValueError:
3382 print("Could not convert label time %s" % labelDetails['Update'])
3383 tmwhen = 1
3385 when = int(time.mktime(tmwhen))
3386 self.streamTag(stream, name, labelDetails, gitCommit, when)
3387 if verbose:
3388 print("p4 label %s mapped to git commit %s" % (name, gitCommit))
3389 else:
3390 if verbose:
3391 print("Label %s has no changelists - possibly deleted?" % name)
3393 if not commitFound:
3394 # We can't import this label; don't try again as it will get very
3395 # expensive repeatedly fetching all the files for labels that will
3396 # never be imported. If the label is moved in the future, the
3397 # ignore will need to be removed manually.
3398 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3400 def guessProjectName(self):
3401 for p in self.depotPaths:
3402 if p.endswith("/"):
3403 p = p[:-1]
3404 p = p[p.strip().rfind("/") + 1:]
3405 if not p.endswith("/"):
3406 p += "/"
3407 return p
3409 def getBranchMapping(self):
3410 lostAndFoundBranches = set()
3412 user = gitConfig("git-p4.branchUser")
3413 if len(user) > 0:
3414 command = "branches -u %s" % user
3415 else:
3416 command = "branches"
3418 for info in p4CmdList(command):
3419 details = p4Cmd(["branch", "-o", info["branch"]])
3420 viewIdx = 0
3421 while "View%s" % viewIdx in details:
3422 paths = details["View%s" % viewIdx].split(" ")
3423 viewIdx = viewIdx + 1
3424 # require standard //depot/foo/... //depot/bar/... mapping
3425 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3426 continue
3427 source = paths[0]
3428 destination = paths[1]
3429 ## HACK
3430 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3431 source = source[len(self.depotPaths[0]):-4]
3432 destination = destination[len(self.depotPaths[0]):-4]
3434 if destination in self.knownBranches:
3435 if not self.silent:
3436 print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3437 print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
3438 continue
3440 self.knownBranches[destination] = source
3442 lostAndFoundBranches.discard(destination)
3444 if source not in self.knownBranches:
3445 lostAndFoundBranches.add(source)
3447 # Perforce does not strictly require branches to be defined, so we also
3448 # check git config for a branch list.
3450 # Example of branch definition in git config file:
3451 # [git-p4]
3452 # branchList=main:branchA
3453 # branchList=main:branchB
3454 # branchList=branchA:branchC
3455 configBranches = gitConfigList("git-p4.branchList")
3456 for branch in configBranches:
3457 if branch:
3458 (source, destination) = branch.split(":")
3459 self.knownBranches[destination] = source
3461 lostAndFoundBranches.discard(destination)
3463 if source not in self.knownBranches:
3464 lostAndFoundBranches.add(source)
3467 for branch in lostAndFoundBranches:
3468 self.knownBranches[branch] = branch
3470 def getBranchMappingFromGitBranches(self):
3471 branches = p4BranchesInGit(self.importIntoRemotes)
3472 for branch in branches.keys():
3473 if branch == "master":
3474 branch = "main"
3475 else:
3476 branch = branch[len(self.projectName):]
3477 self.knownBranches[branch] = branch
3479 def updateOptionDict(self, d):
3480 option_keys = {}
3481 if self.keepRepoPath:
3482 option_keys['keepRepoPath'] = 1
3484 d["options"] = ' '.join(sorted(option_keys.keys()))
3486 def readOptions(self, d):
3487 self.keepRepoPath = ('options' in d
3488 and ('keepRepoPath' in d['options']))
3490 def gitRefForBranch(self, branch):
3491 if branch == "main":
3492 return self.refPrefix + "master"
3494 if len(branch) <= 0:
3495 return branch
3497 return self.refPrefix + self.projectName + branch
3499 def gitCommitByP4Change(self, ref, change):
3500 if self.verbose:
3501 print("looking in ref " + ref + " for change %s using bisect..." % change)
3503 earliestCommit = ""
3504 latestCommit = parseRevision(ref)
3506 while True:
3507 if self.verbose:
3508 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
3509 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3510 if len(next) == 0:
3511 if self.verbose:
3512 print("argh")
3513 return ""
3514 log = extractLogMessageFromGitCommit(next)
3515 settings = extractSettingsGitLog(log)
3516 currentChange = int(settings['change'])
3517 if self.verbose:
3518 print("current change %s" % currentChange)
3520 if currentChange == change:
3521 if self.verbose:
3522 print("found %s" % next)
3523 return next
3525 if currentChange < change:
3526 earliestCommit = "^%s" % next
3527 else:
3528 if next == latestCommit:
3529 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3530 latestCommit = "%s^@" % next
3532 return ""
3534 def importNewBranch(self, branch, maxChange):
3535 # make fast-import flush all changes to disk and update the refs using the checkpoint
3536 # command so that we can try to find the branch parent in the git history
3537 self.gitStream.write("checkpoint\n\n");
3538 self.gitStream.flush();
3539 branchPrefix = self.depotPaths[0] + branch + "/"
3540 range = "@1,%s" % maxChange
3541 #print "prefix" + branchPrefix
3542 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3543 if len(changes) <= 0:
3544 return False
3545 firstChange = changes[0]
3546 #print "first change in branch: %s" % firstChange
3547 sourceBranch = self.knownBranches[branch]
3548 sourceDepotPath = self.depotPaths[0] + sourceBranch
3549 sourceRef = self.gitRefForBranch(sourceBranch)
3550 #print "source " + sourceBranch
3552 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3553 #print "branch parent: %s" % branchParentChange
3554 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3555 if len(gitParent) > 0:
3556 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3557 #print "parent git commit: %s" % gitParent
3559 self.importChanges(changes)
3560 return True
3562 def searchParent(self, parent, branch, target):
3563 parentFound = False
3564 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3565 "--no-merges", parent]):
3566 blob = blob.strip()
3567 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3568 parentFound = True
3569 if self.verbose:
3570 print("Found parent of %s in commit %s" % (branch, blob))
3571 break
3572 if parentFound:
3573 return blob
3574 else:
3575 return None
3577 def importChanges(self, changes, origin_revision=0):
3578 cnt = 1
3579 for change in changes:
3580 description = p4_describe(change)
3581 self.updateOptionDict(description)
3583 if not self.silent:
3584 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3585 sys.stdout.flush()
3586 cnt = cnt + 1
3588 try:
3589 if self.detectBranches:
3590 branches = self.splitFilesIntoBranches(description)
3591 for branch in branches.keys():
3592 ## HACK --hwn
3593 branchPrefix = self.depotPaths[0] + branch + "/"
3594 self.branchPrefixes = [ branchPrefix ]
3596 parent = ""
3598 filesForCommit = branches[branch]
3600 if self.verbose:
3601 print("branch is %s" % branch)
3603 self.updatedBranches.add(branch)
3605 if branch not in self.createdBranches:
3606 self.createdBranches.add(branch)
3607 parent = self.knownBranches[branch]
3608 if parent == branch:
3609 parent = ""
3610 else:
3611 fullBranch = self.projectName + branch
3612 if fullBranch not in self.p4BranchesInGit:
3613 if not self.silent:
3614 print("\n Importing new branch %s" % fullBranch);
3615 if self.importNewBranch(branch, change - 1):
3616 parent = ""
3617 self.p4BranchesInGit.append(fullBranch)
3618 if not self.silent:
3619 print("\n Resuming with change %s" % change);
3621 if self.verbose:
3622 print("parent determined through known branches: %s" % parent)
3624 branch = self.gitRefForBranch(branch)
3625 parent = self.gitRefForBranch(parent)
3627 if self.verbose:
3628 print("looking for initial parent for %s; current parent is %s" % (branch, parent))
3630 if len(parent) == 0 and branch in self.initialParents:
3631 parent = self.initialParents[branch]
3632 del self.initialParents[branch]
3634 blob = None
3635 if len(parent) > 0:
3636 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3637 if self.verbose:
3638 print("Creating temporary branch: " + tempBranch)
3639 self.commit(description, filesForCommit, tempBranch)
3640 self.tempBranches.append(tempBranch)
3641 self.checkpoint()
3642 blob = self.searchParent(parent, branch, tempBranch)
3643 if blob:
3644 self.commit(description, filesForCommit, branch, blob)
3645 else:
3646 if self.verbose:
3647 print("Parent of %s not found. Committing into head of %s" % (branch, parent))
3648 self.commit(description, filesForCommit, branch, parent)
3649 else:
3650 files = self.extractFilesFromCommit(description)
3651 self.commit(description, files, self.branch,
3652 self.initialParent)
3653 # only needed once, to connect to the previous commit
3654 self.initialParent = ""
3655 except IOError:
3656 print(self.gitError.read())
3657 sys.exit(1)
3659 def sync_origin_only(self):
3660 if self.syncWithOrigin:
3661 self.hasOrigin = originP4BranchesExist()
3662 if self.hasOrigin:
3663 if not self.silent:
3664 print('Syncing with origin first, using "git fetch origin"')
3665 system("git fetch origin")
3667 def importHeadRevision(self, revision):
3668 print("Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch))
3670 details = {}
3671 details["user"] = "git perforce import user"
3672 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3673 % (' '.join(self.depotPaths), revision))
3674 details["change"] = revision
3675 newestRevision = 0
3677 fileCnt = 0
3678 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3680 for info in p4CmdList(["files"] + fileArgs):
3682 if 'code' in info and info['code'] == 'error':
3683 sys.stderr.write("p4 returned an error: %s\n"
3684 % info['data'])
3685 if info['data'].find("must refer to client") >= 0:
3686 sys.stderr.write("This particular p4 error is misleading.\n")
3687 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3688 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3689 sys.exit(1)
3690 if 'p4ExitCode' in info:
3691 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3692 sys.exit(1)
3695 change = int(info["change"])
3696 if change > newestRevision:
3697 newestRevision = change
3699 if info["action"] in self.delete_actions:
3700 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3701 #fileCnt = fileCnt + 1
3702 continue
3704 for prop in ["depotFile", "rev", "action", "type" ]:
3705 details["%s%s" % (prop, fileCnt)] = info[prop]
3707 fileCnt = fileCnt + 1
3709 details["change"] = newestRevision
3711 # Use time from top-most change so that all git p4 clones of
3712 # the same p4 repo have the same commit SHA1s.
3713 res = p4_describe(newestRevision)
3714 details["time"] = res["time"]
3716 self.updateOptionDict(details)
3717 try:
3718 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3719 except IOError as err:
3720 print("IO error with git fast-import. Is your git version recent enough?")
3721 print("IO error details: {}".format(err))
3722 print(self.gitError.read())
3725 def importRevisions(self, args, branch_arg_given):
3726 changes = []
3728 if len(self.changesFile) > 0:
3729 with open(self.changesFile) as f:
3730 output = f.readlines()
3731 changeSet = set()
3732 for line in output:
3733 changeSet.add(int(line))
3735 for change in changeSet:
3736 changes.append(change)
3738 changes.sort()
3739 else:
3740 # catch "git p4 sync" with no new branches, in a repo that
3741 # does not have any existing p4 branches
3742 if len(args) == 0:
3743 if not self.p4BranchesInGit:
3744 raise P4CommandException("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3746 # The default branch is master, unless --branch is used to
3747 # specify something else. Make sure it exists, or complain
3748 # nicely about how to use --branch.
3749 if not self.detectBranches:
3750 if not branch_exists(self.branch):
3751 if branch_arg_given:
3752 raise P4CommandException("Error: branch %s does not exist." % self.branch)
3753 else:
3754 raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3755 self.branch)
3757 if self.verbose:
3758 print("Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3759 self.changeRange))
3760 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3762 if len(self.maxChanges) > 0:
3763 changes = changes[:min(int(self.maxChanges), len(changes))]
3765 if len(changes) == 0:
3766 if not self.silent:
3767 print("No changes to import!")
3768 else:
3769 if not self.silent and not self.detectBranches:
3770 print("Import destination: %s" % self.branch)
3772 self.updatedBranches = set()
3774 if not self.detectBranches:
3775 if args:
3776 # start a new branch
3777 self.initialParent = ""
3778 else:
3779 # build on a previous revision
3780 self.initialParent = parseRevision(self.branch)
3782 self.importChanges(changes)
3784 if not self.silent:
3785 print("")
3786 if len(self.updatedBranches) > 0:
3787 sys.stdout.write("Updated branches: ")
3788 for b in self.updatedBranches:
3789 sys.stdout.write("%s " % b)
3790 sys.stdout.write("\n")
3792 def openStreams(self):
3793 self.importProcess = subprocess.Popen(["git", "fast-import"],
3794 stdin=subprocess.PIPE,
3795 stdout=subprocess.PIPE,
3796 stderr=subprocess.PIPE);
3797 self.gitOutput = self.importProcess.stdout
3798 self.gitStream = self.importProcess.stdin
3799 self.gitError = self.importProcess.stderr
3801 if bytes is not str:
3802 # Wrap gitStream.write() so that it can be called using `str` arguments
3803 def make_encoded_write(write):
3804 def encoded_write(s):
3805 return write(s.encode() if isinstance(s, str) else s)
3806 return encoded_write
3808 self.gitStream.write = make_encoded_write(self.gitStream.write)
3810 def closeStreams(self):
3811 if self.gitStream is None:
3812 return
3813 self.gitStream.close()
3814 if self.importProcess.wait() != 0:
3815 die("fast-import failed: %s" % self.gitError.read())
3816 self.gitOutput.close()
3817 self.gitError.close()
3818 self.gitStream = None
3820 def run(self, args):
3821 if self.importIntoRemotes:
3822 self.refPrefix = "refs/remotes/p4/"
3823 else:
3824 self.refPrefix = "refs/heads/p4/"
3826 self.sync_origin_only()
3828 branch_arg_given = bool(self.branch)
3829 if len(self.branch) == 0:
3830 self.branch = self.refPrefix + "master"
3831 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3832 system("git update-ref %s refs/heads/p4" % self.branch)
3833 system("git branch -D p4")
3835 # accept either the command-line option, or the configuration variable
3836 if self.useClientSpec:
3837 # will use this after clone to set the variable
3838 self.useClientSpec_from_options = True
3839 else:
3840 if gitConfigBool("git-p4.useclientspec"):
3841 self.useClientSpec = True
3842 if self.useClientSpec:
3843 self.clientSpecDirs = getClientSpec()
3845 # TODO: should always look at previous commits,
3846 # merge with previous imports, if possible.
3847 if args == []:
3848 if self.hasOrigin:
3849 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3851 # branches holds mapping from branch name to sha1
3852 branches = p4BranchesInGit(self.importIntoRemotes)
3854 # restrict to just this one, disabling detect-branches
3855 if branch_arg_given:
3856 short = self.branch.split("/")[-1]
3857 if short in branches:
3858 self.p4BranchesInGit = [ short ]
3859 else:
3860 self.p4BranchesInGit = branches.keys()
3862 if len(self.p4BranchesInGit) > 1:
3863 if not self.silent:
3864 print("Importing from/into multiple branches")
3865 self.detectBranches = True
3866 for branch in branches.keys():
3867 self.initialParents[self.refPrefix + branch] = \
3868 branches[branch]
3870 if self.verbose:
3871 print("branches: %s" % self.p4BranchesInGit)
3873 p4Change = 0
3874 for branch in self.p4BranchesInGit:
3875 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3877 settings = extractSettingsGitLog(logMsg)
3879 self.readOptions(settings)
3880 if ('depot-paths' in settings
3881 and 'change' in settings):
3882 change = int(settings['change']) + 1
3883 p4Change = max(p4Change, change)
3885 depotPaths = sorted(settings['depot-paths'])
3886 if self.previousDepotPaths == []:
3887 self.previousDepotPaths = depotPaths
3888 else:
3889 paths = []
3890 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3891 prev_list = prev.split("/")
3892 cur_list = cur.split("/")
3893 for i in range(0, min(len(cur_list), len(prev_list))):
3894 if cur_list[i] != prev_list[i]:
3895 i = i - 1
3896 break
3898 paths.append ("/".join(cur_list[:i + 1]))
3900 self.previousDepotPaths = paths
3902 if p4Change > 0:
3903 self.depotPaths = sorted(self.previousDepotPaths)
3904 self.changeRange = "@%s,#head" % p4Change
3905 if not self.silent and not self.detectBranches:
3906 print("Performing incremental import into %s git branch" % self.branch)
3908 # accept multiple ref name abbreviations:
3909 # refs/foo/bar/branch -> use it exactly
3910 # p4/branch -> prepend refs/remotes/ or refs/heads/
3911 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3912 if not self.branch.startswith("refs/"):
3913 if self.importIntoRemotes:
3914 prepend = "refs/remotes/"
3915 else:
3916 prepend = "refs/heads/"
3917 if not self.branch.startswith("p4/"):
3918 prepend += "p4/"
3919 self.branch = prepend + self.branch
3921 if len(args) == 0 and self.depotPaths:
3922 if not self.silent:
3923 print("Depot paths: %s" % ' '.join(self.depotPaths))
3924 else:
3925 if self.depotPaths and self.depotPaths != args:
3926 print("previous import used depot path %s and now %s was specified. "
3927 "This doesn't work!" % (' '.join (self.depotPaths),
3928 ' '.join (args)))
3929 sys.exit(1)
3931 self.depotPaths = sorted(args)
3933 revision = ""
3934 self.users = {}
3936 # Make sure no revision specifiers are used when --changesfile
3937 # is specified.
3938 bad_changesfile = False
3939 if len(self.changesFile) > 0:
3940 for p in self.depotPaths:
3941 if p.find("@") >= 0 or p.find("#") >= 0:
3942 bad_changesfile = True
3943 break
3944 if bad_changesfile:
3945 die("Option --changesfile is incompatible with revision specifiers")
3947 newPaths = []
3948 for p in self.depotPaths:
3949 if p.find("@") != -1:
3950 atIdx = p.index("@")
3951 self.changeRange = p[atIdx:]
3952 if self.changeRange == "@all":
3953 self.changeRange = ""
3954 elif ',' not in self.changeRange:
3955 revision = self.changeRange
3956 self.changeRange = ""
3957 p = p[:atIdx]
3958 elif p.find("#") != -1:
3959 hashIdx = p.index("#")
3960 revision = p[hashIdx:]
3961 p = p[:hashIdx]
3962 elif self.previousDepotPaths == []:
3963 # pay attention to changesfile, if given, else import
3964 # the entire p4 tree at the head revision
3965 if len(self.changesFile) == 0:
3966 revision = "#head"
3968 p = re.sub ("\.\.\.$", "", p)
3969 if not p.endswith("/"):
3970 p += "/"
3972 newPaths.append(p)
3974 self.depotPaths = newPaths
3976 # --detect-branches may change this for each branch
3977 self.branchPrefixes = self.depotPaths
3979 self.loadUserMapFromCache()
3980 self.labels = {}
3981 if self.detectLabels:
3982 self.getLabels();
3984 if self.detectBranches:
3985 ## FIXME - what's a P4 projectName ?
3986 self.projectName = self.guessProjectName()
3988 if self.hasOrigin:
3989 self.getBranchMappingFromGitBranches()
3990 else:
3991 self.getBranchMapping()
3992 if self.verbose:
3993 print("p4-git branches: %s" % self.p4BranchesInGit)
3994 print("initial parents: %s" % self.initialParents)
3995 for b in self.p4BranchesInGit:
3996 if b != "master":
3998 ## FIXME
3999 b = b[len(self.projectName):]
4000 self.createdBranches.add(b)
4002 p4_check_access()
4004 self.openStreams()
4006 err = None
4008 try:
4009 if revision:
4010 self.importHeadRevision(revision)
4011 else:
4012 self.importRevisions(args, branch_arg_given)
4014 if gitConfigBool("git-p4.importLabels"):
4015 self.importLabels = True
4017 if self.importLabels:
4018 p4Labels = getP4Labels(self.depotPaths)
4019 gitTags = getGitTags()
4021 missingP4Labels = p4Labels - gitTags
4022 self.importP4Labels(self.gitStream, missingP4Labels)
4024 except P4CommandException as e:
4025 err = e
4027 finally:
4028 self.closeStreams()
4030 if err:
4031 die(str(err))
4033 # Cleanup temporary branches created during import
4034 if self.tempBranches != []:
4035 for branch in self.tempBranches:
4036 read_pipe("git update-ref -d %s" % branch)
4037 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
4039 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
4040 # a convenient shortcut refname "p4".
4041 if self.importIntoRemotes:
4042 head_ref = self.refPrefix + "HEAD"
4043 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
4044 system(["git", "symbolic-ref", head_ref, self.branch])
4046 return True
4048 class P4Rebase(Command):
4049 def __init__(self):
4050 Command.__init__(self)
4051 self.options = [
4052 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
4054 self.importLabels = False
4055 self.description = ("Fetches the latest revision from perforce and "
4056 + "rebases the current work (branch) against it")
4058 def run(self, args):
4059 sync = P4Sync()
4060 sync.importLabels = self.importLabels
4061 sync.run([])
4063 return self.rebase()
4065 def rebase(self):
4066 if os.system("git update-index --refresh") != 0:
4067 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
4068 if len(read_pipe("git diff-index HEAD --")) > 0:
4069 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
4071 [upstream, settings] = findUpstreamBranchPoint()
4072 if len(upstream) == 0:
4073 die("Cannot find upstream branchpoint for rebase")
4075 # the branchpoint may be p4/foo~3, so strip off the parent
4076 upstream = re.sub("~[0-9]+$", "", upstream)
4078 print("Rebasing the current branch onto %s" % upstream)
4079 oldHead = read_pipe("git rev-parse HEAD").strip()
4080 system("git rebase %s" % upstream)
4081 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
4082 return True
4084 class P4Clone(P4Sync):
4085 def __init__(self):
4086 P4Sync.__init__(self)
4087 self.description = "Creates a new git repository and imports from Perforce into it"
4088 self.usage = "usage: %prog [options] //depot/path[@revRange]"
4089 self.options += [
4090 optparse.make_option("--destination", dest="cloneDestination",
4091 action='store', default=None,
4092 help="where to leave result of the clone"),
4093 optparse.make_option("--bare", dest="cloneBare",
4094 action="store_true", default=False),
4096 self.cloneDestination = None
4097 self.needsGit = False
4098 self.cloneBare = False
4100 def defaultDestination(self, args):
4101 ## TODO: use common prefix of args?
4102 depotPath = args[0]
4103 depotDir = re.sub("(@[^@]*)$", "", depotPath)
4104 depotDir = re.sub("(#[^#]*)$", "", depotDir)
4105 depotDir = re.sub(r"\.\.\.$", "", depotDir)
4106 depotDir = re.sub(r"/$", "", depotDir)
4107 return os.path.split(depotDir)[1]
4109 def run(self, args):
4110 if len(args) < 1:
4111 return False
4113 if self.keepRepoPath and not self.cloneDestination:
4114 sys.stderr.write("Must specify destination for --keep-path\n")
4115 sys.exit(1)
4117 depotPaths = args
4119 if not self.cloneDestination and len(depotPaths) > 1:
4120 self.cloneDestination = depotPaths[-1]
4121 depotPaths = depotPaths[:-1]
4123 for p in depotPaths:
4124 if not p.startswith("//"):
4125 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
4126 return False
4128 if not self.cloneDestination:
4129 self.cloneDestination = self.defaultDestination(args)
4131 print("Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination))
4133 if not os.path.exists(self.cloneDestination):
4134 os.makedirs(self.cloneDestination)
4135 chdir(self.cloneDestination)
4137 init_cmd = [ "git", "init" ]
4138 if self.cloneBare:
4139 init_cmd.append("--bare")
4140 retcode = subprocess.call(init_cmd)
4141 if retcode:
4142 raise CalledProcessError(retcode, init_cmd)
4144 if not P4Sync.run(self, depotPaths):
4145 return False
4147 # create a master branch and check out a work tree
4148 if gitBranchExists(self.branch):
4149 system([ "git", "branch", "master", self.branch ])
4150 if not self.cloneBare:
4151 system([ "git", "checkout", "-f" ])
4152 else:
4153 print('Not checking out any branch, use ' \
4154 '"git checkout -q -b master <branch>"')
4156 # auto-set this variable if invoked with --use-client-spec
4157 if self.useClientSpec_from_options:
4158 system("git config --bool git-p4.useclientspec true")
4160 return True
4162 class P4Unshelve(Command):
4163 def __init__(self):
4164 Command.__init__(self)
4165 self.options = []
4166 self.origin = "HEAD"
4167 self.description = "Unshelve a P4 changelist into a git commit"
4168 self.usage = "usage: %prog [options] changelist"
4169 self.options += [
4170 optparse.make_option("--origin", dest="origin",
4171 help="Use this base revision instead of the default (%s)" % self.origin),
4173 self.verbose = False
4174 self.noCommit = False
4175 self.destbranch = "refs/remotes/p4-unshelved"
4177 def renameBranch(self, branch_name):
4178 """ Rename the existing branch to branch_name.N
4181 found = True
4182 for i in range(0,1000):
4183 backup_branch_name = "{0}.{1}".format(branch_name, i)
4184 if not gitBranchExists(backup_branch_name):
4185 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
4186 gitDeleteRef(branch_name)
4187 found = True
4188 print("renamed old unshelve branch to {0}".format(backup_branch_name))
4189 break
4191 if not found:
4192 sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
4194 def findLastP4Revision(self, starting_point):
4195 """ Look back from starting_point for the first commit created by git-p4
4196 to find the P4 commit we are based on, and the depot-paths.
4199 for parent in (range(65535)):
4200 log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
4201 settings = extractSettingsGitLog(log)
4202 if 'change' in settings:
4203 return settings
4205 sys.exit("could not find git-p4 commits in {0}".format(self.origin))
4207 def createShelveParent(self, change, branch_name, sync, origin):
4208 """ Create a commit matching the parent of the shelved changelist 'change'
4210 parent_description = p4_describe(change, shelved=True)
4211 parent_description['desc'] = 'parent for shelved changelist {}\n'.format(change)
4212 files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)
4214 parent_files = []
4215 for f in files:
4216 # if it was added in the shelved changelist, it won't exist in the parent
4217 if f['action'] in self.add_actions:
4218 continue
4220 # if it was deleted in the shelved changelist it must not be deleted
4221 # in the parent - we might even need to create it if the origin branch
4222 # does not have it
4223 if f['action'] in self.delete_actions:
4224 f['action'] = 'add'
4226 parent_files.append(f)
4228 sync.commit(parent_description, parent_files, branch_name,
4229 parent=origin, allow_empty=True)
4230 print("created parent commit for {0} based on {1} in {2}".format(
4231 change, self.origin, branch_name))
4233 def run(self, args):
4234 if len(args) != 1:
4235 return False
4237 if not gitBranchExists(self.origin):
4238 sys.exit("origin branch {0} does not exist".format(self.origin))
4240 sync = P4Sync()
4241 changes = args
4243 # only one change at a time
4244 change = changes[0]
4246 # if the target branch already exists, rename it
4247 branch_name = "{0}/{1}".format(self.destbranch, change)
4248 if gitBranchExists(branch_name):
4249 self.renameBranch(branch_name)
4250 sync.branch = branch_name
4252 sync.verbose = self.verbose
4253 sync.suppress_meta_comment = True
4255 settings = self.findLastP4Revision(self.origin)
4256 sync.depotPaths = settings['depot-paths']
4257 sync.branchPrefixes = sync.depotPaths
4259 sync.openStreams()
4260 sync.loadUserMapFromCache()
4261 sync.silent = True
4263 # create a commit for the parent of the shelved changelist
4264 self.createShelveParent(change, branch_name, sync, self.origin)
4266 # create the commit for the shelved changelist itself
4267 description = p4_describe(change, True)
4268 files = sync.extractFilesFromCommit(description, True, change)
4270 sync.commit(description, files, branch_name, "")
4271 sync.closeStreams()
4273 print("unshelved changelist {0} into {1}".format(change, branch_name))
4275 return True
4277 class P4Branches(Command):
4278 def __init__(self):
4279 Command.__init__(self)
4280 self.options = [ ]
4281 self.description = ("Shows the git branches that hold imports and their "
4282 + "corresponding perforce depot paths")
4283 self.verbose = False
4285 def run(self, args):
4286 if originP4BranchesExist():
4287 createOrUpdateBranchesFromOrigin()
4289 cmdline = "git rev-parse --symbolic "
4290 cmdline += " --remotes"
4292 for line in read_pipe_lines(cmdline):
4293 line = line.strip()
4295 if not line.startswith('p4/') or line == "p4/HEAD":
4296 continue
4297 branch = line
4299 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4300 settings = extractSettingsGitLog(log)
4302 print("%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]))
4303 return True
4305 class HelpFormatter(optparse.IndentedHelpFormatter):
4306 def __init__(self):
4307 optparse.IndentedHelpFormatter.__init__(self)
4309 def format_description(self, description):
4310 if description:
4311 return description + "\n"
4312 else:
4313 return ""
4315 def printUsage(commands):
4316 print("usage: %s <command> [options]" % sys.argv[0])
4317 print("")
4318 print("valid commands: %s" % ", ".join(commands))
4319 print("")
4320 print("Try %s <command> --help for command specific help." % sys.argv[0])
4321 print("")
4323 commands = {
4324 "debug" : P4Debug,
4325 "submit" : P4Submit,
4326 "commit" : P4Submit,
4327 "sync" : P4Sync,
4328 "rebase" : P4Rebase,
4329 "clone" : P4Clone,
4330 "rollback" : P4RollBack,
4331 "branches" : P4Branches,
4332 "unshelve" : P4Unshelve,
4335 def main():
4336 if len(sys.argv[1:]) == 0:
4337 printUsage(commands.keys())
4338 sys.exit(2)
4340 cmdName = sys.argv[1]
4341 try:
4342 klass = commands[cmdName]
4343 cmd = klass()
4344 except KeyError:
4345 print("unknown command %s" % cmdName)
4346 print("")
4347 printUsage(commands.keys())
4348 sys.exit(2)
4350 options = cmd.options
4351 cmd.gitdir = os.environ.get("GIT_DIR", None)
4353 args = sys.argv[2:]
4355 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4356 if cmd.needsGit:
4357 options.append(optparse.make_option("--git-dir", dest="gitdir"))
4359 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4360 options,
4361 description = cmd.description,
4362 formatter = HelpFormatter())
4364 try:
4365 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4366 except:
4367 parser.print_help()
4368 raise
4370 global verbose
4371 verbose = cmd.verbose
4372 if cmd.needsGit:
4373 if cmd.gitdir == None:
4374 cmd.gitdir = os.path.abspath(".git")
4375 if not isValidGitDir(cmd.gitdir):
4376 # "rev-parse --git-dir" without arguments will try $PWD/.git
4377 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4378 if os.path.exists(cmd.gitdir):
4379 cdup = read_pipe("git rev-parse --show-cdup").strip()
4380 if len(cdup) > 0:
4381 chdir(cdup);
4383 if not isValidGitDir(cmd.gitdir):
4384 if isValidGitDir(cmd.gitdir + "/.git"):
4385 cmd.gitdir += "/.git"
4386 else:
4387 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4389 # so git commands invoked from the P4 workspace will succeed
4390 os.environ["GIT_DIR"] = cmd.gitdir
4392 if not cmd.run(args):
4393 parser.print_help()
4394 sys.exit(2)
4397 if __name__ == '__main__':
4398 main()