3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 # pylint: disable=invalid-name,missing-docstring,too-many-arguments,broad-except
11 # pylint: disable=no-self-use,wrong-import-position,consider-iterating-dictionary
12 # pylint: disable=wrong-import-order,unused-import,too-few-public-methods
13 # pylint: disable=too-many-lines,ungrouped-imports,fixme,too-many-locals
14 # pylint: disable=line-too-long,bad-whitespace,superfluous-parens
15 # pylint: disable=too-many-statements,too-many-instance-attributes
16 # pylint: disable=too-many-branches,too-many-nested-blocks
19 if sys
.version_info
.major
< 3 and sys
.version_info
.minor
< 7:
20 sys
.stderr
.write("git-p4: requires Python 2.7 or later.\n")
39 # On python2.7 where raw_input() and input() are both availble,
40 # we want raw_input's semantics, but aliased to input for python3
42 # support basestring in python3
44 if raw_input and input:
51 # Only labels/tags matching this will be imported/exported
52 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
54 # The block size is reduced automatically if required
55 defaultBlockSize
= 1<<20
57 p4_access_checked
= False
59 re_ko_keywords
= re
.compile(br
'\$(Id|Header)(:[^$\n]+)?\$')
60 re_k_keywords
= re
.compile(br
'\$(Id|Header|Author|Date|DateTime|Change|File|Revision)(:[^$\n]+)?\$')
62 def format_size_human_readable(num
):
63 """ Returns a number of units (typically bytes) formatted as a human-readable
67 return '{:d} B'.format(num
)
68 for unit
in ["Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
71 return "{:3.1f} {}B".format(num
, unit
)
72 return "{:.1f} YiB".format(num
)
74 def p4_build_cmd(cmd
):
75 """Build a suitable p4 command line.
77 This consolidates building and returning a p4 command line into one
78 location. It means that hooking into the environment, or other configuration
79 can be done more easily.
83 user
= gitConfig("git-p4.user")
85 real_cmd
+= ["-u",user
]
87 password
= gitConfig("git-p4.password")
89 real_cmd
+= ["-P", password
]
91 port
= gitConfig("git-p4.port")
93 real_cmd
+= ["-p", port
]
95 host
= gitConfig("git-p4.host")
97 real_cmd
+= ["-H", host
]
99 client
= gitConfig("git-p4.client")
101 real_cmd
+= ["-c", client
]
103 retries
= gitConfigInt("git-p4.retries")
105 # Perform 3 retries by default
108 # Provide a way to not pass this option by setting git-p4.retries to 0
109 real_cmd
+= ["-r", str(retries
)]
113 # now check that we can actually talk to the server
114 global p4_access_checked
115 if not p4_access_checked
:
116 p4_access_checked
= True # suppress access checks in p4_check_access itself
122 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123 This won't automatically add ".git" to a directory.
125 d
= read_pipe(["git", "--git-dir", path
, "rev-parse", "--git-dir"], True).strip()
126 if not d
or len(d
) == 0:
131 def chdir(path
, is_client_path
=False):
132 """Do chdir to the given path, and set the PWD environment
133 variable for use by P4. It does not look at getcwd() output.
134 Since we're not using the shell, it is necessary to set the
135 PWD environment variable explicitly.
137 Normally, expand the path to force it to be absolute. This
138 addresses the use of relative path names inside P4 settings,
139 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
140 as given; it looks for .p4config using PWD.
142 If is_client_path, the path was handed to us directly by p4,
143 and may be a symbolic link. Do not call os.getcwd() in this
144 case, because it will cause p4 to think that PWD is not inside
149 if not is_client_path
:
151 os
.environ
['PWD'] = path
154 """Return free space in bytes on the disk of the given dirname."""
155 if platform
.system() == 'Windows':
156 free_bytes
= ctypes
.c_ulonglong(0)
157 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
158 return free_bytes
.value
160 st
= os
.statvfs(os
.getcwd())
161 return st
.f_bavail
* st
.f_frsize
164 """ Terminate execution. Make sure that any running child processes have been wait()ed for before
170 sys
.stderr
.write(msg
+ "\n")
173 def prompt(prompt_text
):
174 """ Prompt the user to choose one of the choices
176 Choices are identified in the prompt_text by square brackets around
177 a single letter option.
179 choices
= set(m
.group(1) for m
in re
.finditer(r
"\[(.)\]", prompt_text
))
182 sys
.stdout
.write(prompt_text
)
184 response
=sys
.stdin
.readline().strip().lower()
187 response
= response
[0]
188 if response
in choices
:
191 # We need different encoding/decoding strategies for text data being passed
192 # around in pipes depending on python version
194 # For python3, always encode and decode as appropriate
195 def decode_text_stream(s
):
196 return s
.decode() if isinstance(s
, bytes
) else s
197 def encode_text_stream(s
):
198 return s
.encode() if isinstance(s
, str) else s
200 # For python2.7, pass read strings as-is, but also allow writing unicode
201 def decode_text_stream(s
):
203 def encode_text_stream(s
):
204 return s
.encode('utf_8') if isinstance(s
, unicode) else s
206 def decode_path(path
):
207 """Decode a given string (bytes or otherwise) using configured path encoding options
209 encoding
= gitConfig('git-p4.pathEncoding') or 'utf_8'
211 return path
.decode(encoding
, errors
='replace') if isinstance(path
, bytes
) else path
216 path
= path
.decode(encoding
, errors
='replace')
218 print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding
, path
))
221 def run_git_hook(cmd
, param
=[]):
222 """Execute a hook if the hook exists."""
223 args
= ['git', 'hook', 'run', '--ignore-missing', cmd
]
228 return subprocess
.call(args
) == 0
230 def write_pipe(c
, stdin
, *k
, **kw
):
232 sys
.stderr
.write('Writing pipe: {}\n'.format(' '.join(c
)))
234 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, *k
, **kw
)
236 val
= pipe
.write(stdin
)
239 die('Command failed: {}'.format(' '.join(c
)))
243 def p4_write_pipe(c
, stdin
, *k
, **kw
):
244 real_cmd
= p4_build_cmd(c
)
245 if bytes
is not str and isinstance(stdin
, str):
246 stdin
= encode_text_stream(stdin
)
247 return write_pipe(real_cmd
, stdin
, *k
, **kw
)
249 def read_pipe_full(c
, *k
, **kw
):
250 """ Read output from command. Returns a tuple
251 of the return status, stdout text and stderr
255 sys
.stderr
.write('Reading pipe: {}\n'.format(' '.join(c
)))
257 p
= subprocess
.Popen(
258 c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, *k
, **kw
)
259 (out
, err
) = p
.communicate()
260 return (p
.returncode
, out
, decode_text_stream(err
))
262 def read_pipe(c
, ignore_error
=False, raw
=False, *k
, **kw
):
263 """ Read output from command. Returns the output text on
264 success. On failure, terminates execution, unless
265 ignore_error is True, when it returns an empty string.
267 If raw is True, do not attempt to decode output text.
269 (retcode
, out
, err
) = read_pipe_full(c
, *k
, **kw
)
274 die('Command failed: {}\nError: {}'.format(' '.join(c
), err
))
276 out
= decode_text_stream(out
)
279 def read_pipe_text(c
, *k
, **kw
):
280 """ Read output from a command with trailing whitespace stripped.
281 On error, returns None.
283 (retcode
, out
, err
) = read_pipe_full(c
, *k
, **kw
)
287 return decode_text_stream(out
).rstrip()
289 def p4_read_pipe(c
, ignore_error
=False, raw
=False, *k
, **kw
):
290 real_cmd
= p4_build_cmd(c
)
291 return read_pipe(real_cmd
, ignore_error
, raw
=raw
, *k
, **kw
)
293 def read_pipe_lines(c
, raw
=False, *k
, **kw
):
295 sys
.stderr
.write('Reading pipe: {}\n'.format(' '.join(c
)))
297 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, *k
, **kw
)
299 lines
= pipe
.readlines()
301 lines
= [decode_text_stream(line
) for line
in lines
]
302 if pipe
.close() or p
.wait():
303 die('Command failed: {}'.format(' '.join(c
)))
306 def p4_read_pipe_lines(c
, *k
, **kw
):
307 """Specifically invoke p4 on the command supplied. """
308 real_cmd
= p4_build_cmd(c
)
309 return read_pipe_lines(real_cmd
, *k
, **kw
)
311 def p4_has_command(cmd
):
312 """Ask p4 for help on this command. If it returns an error, the
313 command does not exist in this version of p4."""
314 real_cmd
= p4_build_cmd(["help", cmd
])
315 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
316 stderr
=subprocess
.PIPE
)
318 return p
.returncode
== 0
320 def p4_has_move_command():
321 """See if the move command exists, that it supports -k, and that
322 it has not been administratively disabled. The arguments
323 must be correct, but the filenames do not have to exist. Use
324 ones with wildcards so even if they exist, it will fail."""
326 if not p4_has_command("move"):
328 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
329 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
330 (out
, err
) = p
.communicate()
331 err
= decode_text_stream(err
)
332 # return code will be 1 in either case
333 if err
.find("Invalid option") >= 0:
335 if err
.find("disabled") >= 0:
337 # assume it failed because @... was invalid changelist
340 def system(cmd
, ignore_error
=False, *k
, **kw
):
342 sys
.stderr
.write("executing {}\n".format(
343 ' '.join(cmd
) if isinstance(cmd
, list) else cmd
))
344 retcode
= subprocess
.call(cmd
, *k
, **kw
)
345 if retcode
and not ignore_error
:
346 raise subprocess
.CalledProcessError(retcode
, cmd
)
350 def p4_system(cmd
, *k
, **kw
):
351 """Specifically invoke p4 as the system command. """
352 real_cmd
= p4_build_cmd(cmd
)
353 retcode
= subprocess
.call(real_cmd
, *k
, **kw
)
355 raise subprocess
.CalledProcessError(retcode
, real_cmd
)
357 def die_bad_access(s
):
358 die("failure accessing depot: {0}".format(s
.rstrip()))
360 def p4_check_access(min_expiration
=1):
361 """ Check if we can access Perforce - account still logged in
363 results
= p4CmdList(["login", "-s"])
365 if len(results
) == 0:
366 # should never get here: always get either some results, or a p4ExitCode
367 assert("could not parse response from perforce")
371 if 'p4ExitCode' in result
:
372 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
373 die_bad_access("could not run p4")
375 code
= result
.get("code")
377 # we get here if we couldn't connect and there was nothing to unmarshal
378 die_bad_access("could not connect")
381 expiry
= result
.get("TicketExpiration")
384 if expiry
> min_expiration
:
388 die_bad_access("perforce ticket expires in {0} seconds".format(expiry
))
391 # account without a timeout - all ok
394 elif code
== "error":
395 data
= result
.get("data")
397 die_bad_access("p4 error: {0}".format(data
))
399 die_bad_access("unknown error")
403 die_bad_access("unknown error code {0}".format(code
))
405 _p4_version_string
= None
406 def p4_version_string():
407 """Read the version string, showing just the last line, which
408 hopefully is the interesting version bit.
411 Perforce - The Fast Software Configuration Management System.
412 Copyright 1995-2011 Perforce Software. All rights reserved.
413 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
415 global _p4_version_string
416 if not _p4_version_string
:
417 a
= p4_read_pipe_lines(["-V"])
418 _p4_version_string
= a
[-1].rstrip()
419 return _p4_version_string
421 def p4_integrate(src
, dest
):
422 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
424 def p4_sync(f
, *options
):
425 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
428 # forcibly add file names with wildcards
429 if wildcard_present(f
):
430 p4_system(["add", "-f", f
])
432 p4_system(["add", f
])
435 p4_system(["delete", wildcard_encode(f
)])
437 def p4_edit(f
, *options
):
438 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
441 p4_system(["revert", wildcard_encode(f
)])
443 def p4_reopen(type, f
):
444 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
446 def p4_reopen_in_change(changelist
, files
):
447 cmd
= ["reopen", "-c", str(changelist
)] + files
450 def p4_move(src
, dest
):
451 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
453 def p4_last_change():
454 results
= p4CmdList(["changes", "-m", "1"], skip_info
=True)
455 return int(results
[0]['change'])
457 def p4_describe(change
, shelved
=False):
458 """Make sure it returns a valid result by checking for
459 the presence of field "time". Return a dict of the
462 cmd
= ["describe", "-s"]
467 ds
= p4CmdList(cmd
, skip_info
=True)
469 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
473 if "p4ExitCode" in d
:
474 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
477 if d
["code"] == "error":
478 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
481 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
486 # Canonicalize the p4 type and return a tuple of the
487 # base type, plus any modifiers. See "p4 help filetypes"
488 # for a list and explanation.
490 def split_p4_type(p4type
):
492 p4_filetypes_historical
= {
493 "ctempobj": "binary+Sw",
499 "tempobj": "binary+FSw",
500 "ubinary": "binary+F",
501 "uresource": "resource+F",
502 "uxbinary": "binary+Fx",
503 "xbinary": "binary+x",
505 "xtempobj": "binary+Swx",
507 "xunicode": "unicode+x",
510 if p4type
in p4_filetypes_historical
:
511 p4type
= p4_filetypes_historical
[p4type
]
513 s
= p4type
.split("+")
521 # return the raw p4 type of a file (text, text+ko, etc)
524 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
525 return results
[0]['headType']
528 # Given a type base and modifier, return a regexp matching
529 # the keywords that can be expanded in the file
531 def p4_keywords_regexp_for_type(base
, type_mods
):
532 if base
in ("text", "unicode", "binary"):
533 if "ko" in type_mods
:
534 return re_ko_keywords
535 elif "k" in type_mods
:
543 # Given a file, return a regexp matching the possible
544 # RCS keywords that will be expanded, or None for files
545 # with kw expansion turned off.
547 def p4_keywords_regexp_for_file(file):
548 if not os
.path
.exists(file):
551 (type_base
, type_mods
) = split_p4_type(p4_type(file))
552 return p4_keywords_regexp_for_type(type_base
, type_mods
)
554 def setP4ExecBit(file, mode
):
555 # Reopens an already open file and changes the execute bit to match
556 # the execute bit setting in the passed in mode.
560 if not isModeExec(mode
):
561 p4Type
= getP4OpenedType(file)
562 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
563 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
564 if p4Type
[-1] == "+":
565 p4Type
= p4Type
[0:-1]
567 p4_reopen(p4Type
, file)
569 def getP4OpenedType(file):
570 # Returns the perforce file type for the given file.
572 result
= p4_read_pipe(["opened", wildcard_encode(file)])
573 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
575 return match
.group(1)
577 die("Could not determine file type for %s (result: '%s')" % (file, result
))
579 # Return the set of all p4 labels
580 def getP4Labels(depotPaths
):
582 if not isinstance(depotPaths
, list):
583 depotPaths
= [depotPaths
]
585 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
591 # Return the set of all git tags
594 for line
in read_pipe_lines(["git", "tag"]):
599 _diff_tree_pattern
= None
601 def parseDiffTreeEntry(entry
):
602 """Parses a single diff tree entry into its component elements.
604 See git-diff-tree(1) manpage for details about the format of the diff
605 output. This method returns a dictionary with the following elements:
607 src_mode - The mode of the source file
608 dst_mode - The mode of the destination file
609 src_sha1 - The sha1 for the source file
610 dst_sha1 - The sha1 fr the destination file
611 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
612 status_score - The score for the status (applicable for 'C' and 'R'
613 statuses). This is None if there is no score.
614 src - The path for the source file.
615 dst - The path for the destination file. This is only present for
616 copy or renames. If it is not present, this is None.
618 If the pattern is not matched, None is returned."""
620 global _diff_tree_pattern
621 if not _diff_tree_pattern
:
622 _diff_tree_pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
624 match
= _diff_tree_pattern
.match(entry
)
627 'src_mode': match
.group(1),
628 'dst_mode': match
.group(2),
629 'src_sha1': match
.group(3),
630 'dst_sha1': match
.group(4),
631 'status': match
.group(5),
632 'status_score': match
.group(6),
633 'src': match
.group(7),
634 'dst': match
.group(10)
638 def isModeExec(mode
):
639 # Returns True if the given git mode represents an executable file,
641 return mode
[-3:] == "755"
643 class P4Exception(Exception):
644 """ Base class for exceptions from the p4 client """
645 def __init__(self
, exit_code
):
646 self
.p4ExitCode
= exit_code
648 class P4ServerException(P4Exception
):
649 """ Base class for exceptions where we get some kind of marshalled up result from the server """
650 def __init__(self
, exit_code
, p4_result
):
651 super(P4ServerException
, self
).__init
__(exit_code
)
652 self
.p4_result
= p4_result
653 self
.code
= p4_result
[0]['code']
654 self
.data
= p4_result
[0]['data']
656 class P4RequestSizeException(P4ServerException
):
657 """ One of the maxresults or maxscanrows errors """
658 def __init__(self
, exit_code
, p4_result
, limit
):
659 super(P4RequestSizeException
, self
).__init
__(exit_code
, p4_result
)
662 class P4CommandException(P4Exception
):
663 """ Something went wrong calling p4 which means we have to give up """
664 def __init__(self
, msg
):
670 def isModeExecChanged(src_mode
, dst_mode
):
671 return isModeExec(src_mode
) != isModeExec(dst_mode
)
673 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None, skip_info
=False,
674 errors_as_exceptions
=False, *k
, **kw
):
676 cmd
= p4_build_cmd(["-G"] + cmd
)
678 sys
.stderr
.write("Opening pipe: {}\n".format(' '.join(cmd
)))
680 # Use a temporary file to avoid deadlocks without
681 # subprocess.communicate(), which would put another copy
682 # of stdout into memory.
684 if stdin
is not None:
685 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
686 if not isinstance(stdin
, list):
687 stdin_file
.write(stdin
)
690 stdin_file
.write(encode_text_stream(i
))
691 stdin_file
.write(b
'\n')
695 p4
= subprocess
.Popen(
696 cmd
, stdin
=stdin_file
, stdout
=subprocess
.PIPE
, *k
, **kw
)
701 entry
= marshal
.load(p4
.stdout
)
703 # Decode unmarshalled dict to use str keys and values, except for:
704 # - `data` which may contain arbitrary binary data
705 # - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
707 for key
, value
in entry
.items():
709 if isinstance(value
, bytes
) and not (key
in ('data', 'path', 'clientFile') or key
.startswith('depotFile')):
710 value
= value
.decode()
711 decoded_entry
[key
] = value
712 # Parse out data if it's an error response
713 if decoded_entry
.get('code') == 'error' and 'data' in decoded_entry
:
714 decoded_entry
['data'] = decoded_entry
['data'].decode()
715 entry
= decoded_entry
717 if 'code' in entry
and entry
['code'] == 'info':
727 if errors_as_exceptions
:
729 data
= result
[0].get('data')
731 m
= re
.search('Too many rows scanned \(over (\d+)\)', data
)
733 m
= re
.search('Request too large \(over (\d+)\)', data
)
736 limit
= int(m
.group(1))
737 raise P4RequestSizeException(exitCode
, result
, limit
)
739 raise P4ServerException(exitCode
, result
)
741 raise P4Exception(exitCode
)
744 entry
["p4ExitCode"] = exitCode
749 def p4Cmd(cmd
, *k
, **kw
):
750 list = p4CmdList(cmd
, *k
, **kw
)
756 def p4Where(depotPath
):
757 if not depotPath
.endswith("/"):
759 depotPathLong
= depotPath
+ "..."
760 outputList
= p4CmdList(["where", depotPathLong
])
762 for entry
in outputList
:
763 if "depotFile" in entry
:
764 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
765 # The base path always ends with "/...".
766 entry_path
= decode_path(entry
['depotFile'])
767 if entry_path
.find(depotPath
) == 0 and entry_path
[-4:] == "/...":
770 elif "data" in entry
:
771 data
= entry
.get("data")
772 space
= data
.find(" ")
773 if data
[:space
] == depotPath
:
778 if output
["code"] == "error":
782 clientPath
= decode_path(output
['path'])
783 elif "data" in output
:
784 data
= output
.get("data")
785 lastSpace
= data
.rfind(b
" ")
786 clientPath
= decode_path(data
[lastSpace
+ 1:])
788 if clientPath
.endswith("..."):
789 clientPath
= clientPath
[:-3]
792 def currentGitBranch():
793 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
795 def isValidGitDir(path
):
796 return git_dir(path
) != None
798 def parseRevision(ref
):
799 return read_pipe(["git", "rev-parse", ref
]).strip()
801 def branchExists(ref
):
802 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
806 def extractLogMessageFromGitCommit(commit
):
809 ## fixme: title is first line of commit, not 1st paragraph.
811 for log
in read_pipe_lines(["git", "cat-file", "commit", commit
]):
820 def extractSettingsGitLog(log
):
822 for line
in log
.split("\n"):
824 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
828 assignments
= m
.group(1).split (':')
829 for a
in assignments
:
831 key
= vals
[0].strip()
832 val
= ('='.join (vals
[1:])).strip()
833 if val
.endswith ('\"') and val
.startswith('"'):
838 paths
= values
.get("depot-paths")
840 paths
= values
.get("depot-path")
842 values
['depot-paths'] = paths
.split(',')
845 def gitBranchExists(branch
):
846 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
847 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
848 return proc
.wait() == 0;
850 def gitUpdateRef(ref
, newvalue
):
851 subprocess
.check_call(["git", "update-ref", ref
, newvalue
])
853 def gitDeleteRef(ref
):
854 subprocess
.check_call(["git", "update-ref", "-d", ref
])
858 def gitConfig(key
, typeSpecifier
=None):
859 if key
not in _gitConfig
:
860 cmd
= [ "git", "config" ]
862 cmd
+= [ typeSpecifier
]
864 s
= read_pipe(cmd
, ignore_error
=True)
865 _gitConfig
[key
] = s
.strip()
866 return _gitConfig
[key
]
868 def gitConfigBool(key
):
869 """Return a bool, using git config --bool. It is True only if the
870 variable is set to true, and False if set to false or not present
873 if key
not in _gitConfig
:
874 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
875 return _gitConfig
[key
]
877 def gitConfigInt(key
):
878 if key
not in _gitConfig
:
879 cmd
= [ "git", "config", "--int", key
]
880 s
= read_pipe(cmd
, ignore_error
=True)
883 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
885 _gitConfig
[key
] = None
886 return _gitConfig
[key
]
888 def gitConfigList(key
):
889 if key
not in _gitConfig
:
890 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
891 _gitConfig
[key
] = s
.strip().splitlines()
892 if _gitConfig
[key
] == ['']:
894 return _gitConfig
[key
]
896 def fullP4Ref(incomingRef
, importIntoRemotes
=True):
897 """Standardize a given provided p4 ref value to a full git ref:
898 refs/foo/bar/branch -> use it exactly
899 p4/branch -> prepend refs/remotes/ or refs/heads/
900 branch -> prepend refs/remotes/p4/ or refs/heads/p4/"""
901 if incomingRef
.startswith("refs/"):
903 if importIntoRemotes
:
904 prepend
= "refs/remotes/"
906 prepend
= "refs/heads/"
907 if not incomingRef
.startswith("p4/"):
909 return prepend
+ incomingRef
911 def shortP4Ref(incomingRef
, importIntoRemotes
=True):
912 """Standardize to a "short ref" if possible:
913 refs/foo/bar/branch -> ignore
914 refs/remotes/p4/branch or refs/heads/p4/branch -> shorten
915 p4/branch -> shorten"""
916 if importIntoRemotes
:
917 longprefix
= "refs/remotes/p4/"
919 longprefix
= "refs/heads/p4/"
920 if incomingRef
.startswith(longprefix
):
921 return incomingRef
[len(longprefix
):]
922 if incomingRef
.startswith("p4/"):
923 return incomingRef
[3:]
926 def p4BranchesInGit(branchesAreInRemotes
=True):
927 """Find all the branches whose names start with "p4/", looking
928 in remotes or heads as specified by the argument. Return
929 a dictionary of { branch: revision } for each one found.
930 The branch names are the short names, without any
935 cmdline
= ["git", "rev-parse", "--symbolic"]
936 if branchesAreInRemotes
:
937 cmdline
.append("--remotes")
939 cmdline
.append("--branches")
941 for line
in read_pipe_lines(cmdline
):
945 if not line
.startswith('p4/'):
947 # special symbolic ref to p4/master
948 if line
== "p4/HEAD":
951 # strip off p4/ prefix
952 branch
= line
[len("p4/"):]
954 branches
[branch
] = parseRevision(line
)
958 def branch_exists(branch
):
959 """Make sure that the given ref name really exists."""
961 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
962 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
963 out
, _
= p
.communicate()
964 out
= decode_text_stream(out
)
967 # expect exactly one line of output: the branch name
968 return out
.rstrip() == branch
970 def findUpstreamBranchPoint(head
= "HEAD"):
971 branches
= p4BranchesInGit()
972 # map from depot-path to branch name
973 branchByDepotPath
= {}
974 for branch
in branches
.keys():
975 tip
= branches
[branch
]
976 log
= extractLogMessageFromGitCommit(tip
)
977 settings
= extractSettingsGitLog(log
)
978 if "depot-paths" in settings
:
979 paths
= ",".join(settings
["depot-paths"])
980 branchByDepotPath
[paths
] = "remotes/p4/" + branch
984 while parent
< 65535:
985 commit
= head
+ "~%s" % parent
986 log
= extractLogMessageFromGitCommit(commit
)
987 settings
= extractSettingsGitLog(log
)
988 if "depot-paths" in settings
:
989 paths
= ",".join(settings
["depot-paths"])
990 if paths
in branchByDepotPath
:
991 return [branchByDepotPath
[paths
], settings
]
995 return ["", settings
]
997 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
999 print("Creating/updating branch(es) in %s based on origin branch(es)"
1002 originPrefix
= "origin/p4/"
1004 for line
in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
1006 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
1009 headName
= line
[len(originPrefix
):]
1010 remoteHead
= localRefPrefix
+ headName
1013 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
1014 if ('depot-paths' not in original
1015 or 'change' not in original
):
1019 if not gitBranchExists(remoteHead
):
1021 print("creating %s" % remoteHead
)
1024 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
1025 if 'change' in settings
:
1026 if settings
['depot-paths'] == original
['depot-paths']:
1027 originP4Change
= int(original
['change'])
1028 p4Change
= int(settings
['change'])
1029 if originP4Change
> p4Change
:
1030 print("%s (%s) is newer than %s (%s). "
1031 "Updating p4 branch from origin."
1032 % (originHead
, originP4Change
,
1033 remoteHead
, p4Change
))
1036 print("Ignoring: %s was imported from %s while "
1037 "%s was imported from %s"
1038 % (originHead
, ','.join(original
['depot-paths']),
1039 remoteHead
, ','.join(settings
['depot-paths'])))
1042 system(["git", "update-ref", remoteHead
, originHead
])
1044 def originP4BranchesExist():
1045 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
1048 def p4ParseNumericChangeRange(parts
):
1049 changeStart
= int(parts
[0][1:])
1050 if parts
[1] == '#head':
1051 changeEnd
= p4_last_change()
1053 changeEnd
= int(parts
[1])
1055 return (changeStart
, changeEnd
)
1057 def chooseBlockSize(blockSize
):
1061 return defaultBlockSize
1063 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
1066 # Parse the change range into start and end. Try to find integer
1067 # revision ranges as these can be broken up into blocks to avoid
1068 # hitting server-side limits (maxrows, maxscanresults). But if
1069 # that doesn't work, fall back to using the raw revision specifier
1070 # strings, without using block mode.
1072 if changeRange
is None or changeRange
== '':
1074 changeEnd
= p4_last_change()
1075 block_size
= chooseBlockSize(requestedBlockSize
)
1077 parts
= changeRange
.split(',')
1078 assert len(parts
) == 2
1080 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
1081 block_size
= chooseBlockSize(requestedBlockSize
)
1083 changeStart
= parts
[0][1:]
1084 changeEnd
= parts
[1]
1085 if requestedBlockSize
:
1086 die("cannot use --changes-block-size with non-numeric revisions")
1091 # Retrieve changes a block at a time, to prevent running
1092 # into a MaxResults/MaxScanRows error from the server. If
1093 # we _do_ hit one of those errors, turn down the block size
1099 end
= min(changeEnd
, changeStart
+ block_size
)
1100 revisionRange
= "%d,%d" % (changeStart
, end
)
1102 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
1104 for p
in depotPaths
:
1105 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
1109 result
= p4CmdList(cmd
, errors_as_exceptions
=True)
1110 except P4RequestSizeException
as e
:
1112 block_size
= e
.limit
1113 elif block_size
> e
.limit
:
1114 block_size
= e
.limit
1116 block_size
= max(2, block_size
// 2)
1118 if verbose
: print("block size error, retrying with block size {0}".format(block_size
))
1120 except P4Exception
as e
:
1121 die('Error retrieving changes description ({0})'.format(e
.p4ExitCode
))
1123 # Insert changes in chronological order
1124 for entry
in reversed(result
):
1125 if 'change' not in entry
:
1127 changes
.add(int(entry
['change']))
1132 if end
>= changeEnd
:
1135 changeStart
= end
+ 1
1137 changes
= sorted(changes
)
1140 def p4PathStartsWith(path
, prefix
):
1141 # This method tries to remedy a potential mixed-case issue:
1143 # If UserA adds //depot/DirA/file1
1144 # and UserB adds //depot/dira/file2
1146 # we may or may not have a problem. If you have core.ignorecase=true,
1147 # we treat DirA and dira as the same directory
1148 if gitConfigBool("core.ignorecase"):
1149 return path
.lower().startswith(prefix
.lower())
1150 return path
.startswith(prefix
)
1152 def getClientSpec():
1153 """Look at the p4 client spec, create a View() object that contains
1154 all the mappings, and return it."""
1156 specList
= p4CmdList(["client", "-o"])
1157 if len(specList
) != 1:
1158 die('Output from "client -o" is %d lines, expecting 1' %
1161 # dictionary of all client parameters
1164 # the //client/ name
1165 client_name
= entry
["Client"]
1167 # just the keys that start with "View"
1168 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
1170 # hold this new View
1171 view
= View(client_name
)
1173 # append the lines, in order, to the view
1174 for view_num
in range(len(view_keys
)):
1175 k
= "View%d" % view_num
1176 if k
not in view_keys
:
1177 die("Expected view key %s missing" % k
)
1178 view
.append(entry
[k
])
1182 def getClientRoot():
1183 """Grab the client directory."""
1185 output
= p4CmdList(["client", "-o"])
1186 if len(output
) != 1:
1187 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
1190 if "Root" not in entry
:
1191 die('Client has no "Root"')
1193 return entry
["Root"]
1196 # P4 wildcards are not allowed in filenames. P4 complains
1197 # if you simply add them, but you can force it with "-f", in
1198 # which case it translates them into %xx encoding internally.
1200 def wildcard_decode(path
):
1201 # Search for and fix just these four characters. Do % last so
1202 # that fixing it does not inadvertently create new %-escapes.
1203 # Cannot have * in a filename in windows; untested as to
1204 # what p4 would do in such a case.
1205 if not platform
.system() == "Windows":
1206 path
= path
.replace("%2A", "*")
1207 path
= path
.replace("%23", "#") \
1208 .replace("%40", "@") \
1209 .replace("%25", "%")
1212 def wildcard_encode(path
):
1213 # do % first to avoid double-encoding the %s introduced here
1214 path
= path
.replace("%", "%25") \
1215 .replace("*", "%2A") \
1216 .replace("#", "%23") \
1217 .replace("@", "%40")
1220 def wildcard_present(path
):
1221 m
= re
.search("[*#@%]", path
)
1222 return m
is not None
1224 class LargeFileSystem(object):
1225 """Base class for large file system support."""
1227 def __init__(self
, writeToGitStream
):
1228 self
.largeFiles
= set()
1229 self
.writeToGitStream
= writeToGitStream
1231 def generatePointer(self
, cloneDestination
, contentFile
):
1232 """Return the content of a pointer file that is stored in Git instead of
1233 the actual content."""
1234 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
1236 def pushFile(self
, localLargeFile
):
1237 """Push the actual content which is not stored in the Git repository to
1239 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
1241 def hasLargeFileExtension(self
, relPath
):
1242 return functools
.reduce(
1243 lambda a
, b
: a
or b
,
1244 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
1248 def generateTempFile(self
, contents
):
1249 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1251 contentFile
.write(d
)
1253 return contentFile
.name
1255 def exceedsLargeFileThreshold(self
, relPath
, contents
):
1256 if gitConfigInt('git-p4.largeFileThreshold'):
1257 contentsSize
= sum(len(d
) for d
in contents
)
1258 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
1260 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1261 contentsSize
= sum(len(d
) for d
in contents
)
1262 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1264 contentTempFile
= self
.generateTempFile(contents
)
1265 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=True)
1266 with zipfile
.ZipFile(compressedContentFile
, mode
='w') as zf
:
1267 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
1268 compressedContentsSize
= zf
.infolist()[0].compress_size
1269 os
.remove(contentTempFile
)
1270 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
1274 def addLargeFile(self
, relPath
):
1275 self
.largeFiles
.add(relPath
)
1277 def removeLargeFile(self
, relPath
):
1278 self
.largeFiles
.remove(relPath
)
1280 def isLargeFile(self
, relPath
):
1281 return relPath
in self
.largeFiles
1283 def processContent(self
, git_mode
, relPath
, contents
):
1284 """Processes the content of git fast import. This method decides if a
1285 file is stored in the large file system and handles all necessary
1287 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1288 contentTempFile
= self
.generateTempFile(contents
)
1289 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1290 if pointer_git_mode
:
1291 git_mode
= pointer_git_mode
1293 # Move temp file to final location in large file system
1294 largeFileDir
= os
.path
.dirname(localLargeFile
)
1295 if not os
.path
.isdir(largeFileDir
):
1296 os
.makedirs(largeFileDir
)
1297 shutil
.move(contentTempFile
, localLargeFile
)
1298 self
.addLargeFile(relPath
)
1299 if gitConfigBool('git-p4.largeFilePush'):
1300 self
.pushFile(localLargeFile
)
1302 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1303 return (git_mode
, contents
)
1305 class MockLFS(LargeFileSystem
):
1306 """Mock large file system for testing."""
1308 def generatePointer(self
, contentFile
):
1309 """The pointer content is the original content prefixed with "pointer-".
1310 The local filename of the large file storage is derived from the file content.
1312 with
open(contentFile
, 'r') as f
:
1315 pointerContents
= 'pointer-' + content
1316 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1317 return (gitMode
, pointerContents
, localLargeFile
)
1319 def pushFile(self
, localLargeFile
):
1320 """The remote filename of the large file storage is the same as the local
1321 one but in a different directory.
1323 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1324 if not os
.path
.exists(remotePath
):
1325 os
.makedirs(remotePath
)
1326 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1328 class GitLFS(LargeFileSystem
):
1329 """Git LFS as backend for the git-p4 large file system.
1330 See https://git-lfs.github.com/ for details."""
1332 def __init__(self
, *args
):
1333 LargeFileSystem
.__init
__(self
, *args
)
1334 self
.baseGitAttributes
= []
1336 def generatePointer(self
, contentFile
):
1337 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1338 mode and content which is stored in the Git repository instead of
1339 the actual content. Return also the new location of the actual
1342 if os
.path
.getsize(contentFile
) == 0:
1343 return (None, '', None)
1345 pointerProcess
= subprocess
.Popen(
1346 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1347 stdout
=subprocess
.PIPE
1349 pointerFile
= decode_text_stream(pointerProcess
.stdout
.read())
1350 if pointerProcess
.wait():
1351 os
.remove(contentFile
)
1352 die('git-lfs pointer command failed. Did you install the extension?')
1354 # Git LFS removed the preamble in the output of the 'pointer' command
1355 # starting from version 1.2.0. Check for the preamble here to support
1357 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1358 if pointerFile
.startswith('Git LFS pointer for'):
1359 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1361 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1362 # if someone use external lfs.storage ( not in local repo git )
1363 lfs_path
= gitConfig('lfs.storage')
1366 if not os
.path
.isabs(lfs_path
):
1367 lfs_path
= os
.path
.join(os
.getcwd(), '.git', lfs_path
)
1368 localLargeFile
= os
.path
.join(
1370 'objects', oid
[:2], oid
[2:4],
1373 # LFS Spec states that pointer files should not have the executable bit set.
1375 return (gitMode
, pointerFile
, localLargeFile
)
1377 def pushFile(self
, localLargeFile
):
1378 uploadProcess
= subprocess
.Popen(
1379 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1381 if uploadProcess
.wait():
1382 die('git-lfs push command failed. Did you define a remote?')
1384 def generateGitAttributes(self
):
1386 self
.baseGitAttributes
+
1390 '# Git LFS (see https://git-lfs.github.com/)\n',
1393 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1394 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1396 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1397 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1401 def addLargeFile(self
, relPath
):
1402 LargeFileSystem
.addLargeFile(self
, relPath
)
1403 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1405 def removeLargeFile(self
, relPath
):
1406 LargeFileSystem
.removeLargeFile(self
, relPath
)
1407 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1409 def processContent(self
, git_mode
, relPath
, contents
):
1410 if relPath
== '.gitattributes':
1411 self
.baseGitAttributes
= contents
1412 return (git_mode
, self
.generateGitAttributes())
1414 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1417 delete_actions
= ( "delete", "move/delete", "purge" )
1418 add_actions
= ( "add", "branch", "move/add" )
1421 self
.usage
= "usage: %prog [options]"
1422 self
.needsGit
= True
1423 self
.verbose
= False
1425 # This is required for the "append" update_shelve action
1426 def ensure_value(self
, attr
, value
):
1427 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
1428 setattr(self
, attr
, value
)
1429 return getattr(self
, attr
)
1433 self
.userMapFromPerforceServer
= False
1434 self
.myP4UserId
= None
1438 return self
.myP4UserId
1440 results
= p4CmdList(["user", "-o"])
1443 self
.myP4UserId
= r
['User']
1445 die("Could not find your p4 user id")
1447 def p4UserIsMe(self
, p4User
):
1448 # return True if the given p4 user is actually me
1449 me
= self
.p4UserId()
1450 if not p4User
or p4User
!= me
:
1455 def getUserCacheFilename(self
):
1456 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1457 return home
+ "/.gitp4-usercache.txt"
1459 def getUserMapFromPerforceServer(self
):
1460 if self
.userMapFromPerforceServer
:
1465 for output
in p4CmdList(["users"]):
1466 if "User" not in output
:
1468 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1469 self
.emails
[output
["Email"]] = output
["User"]
1471 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1472 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1473 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1474 if mapUser
and len(mapUser
[0]) == 3:
1475 user
= mapUser
[0][0]
1476 fullname
= mapUser
[0][1]
1477 email
= mapUser
[0][2]
1478 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1479 self
.emails
[email
] = user
1482 for (key
, val
) in self
.users
.items():
1483 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1485 open(self
.getUserCacheFilename(), 'w').write(s
)
1486 self
.userMapFromPerforceServer
= True
1488 def loadUserMapFromCache(self
):
1490 self
.userMapFromPerforceServer
= False
1492 cache
= open(self
.getUserCacheFilename(), 'r')
1493 lines
= cache
.readlines()
1496 entry
= line
.strip().split("\t")
1497 self
.users
[entry
[0]] = entry
[1]
1499 self
.getUserMapFromPerforceServer()
1501 class P4Submit(Command
, P4UserMap
):
1503 conflict_behavior_choices
= ("ask", "skip", "quit")
1506 Command
.__init
__(self
)
1507 P4UserMap
.__init
__(self
)
1509 optparse
.make_option("--origin", dest
="origin"),
1510 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1511 # preserve the user, requires relevant p4 permissions
1512 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1513 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1514 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1515 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1516 optparse
.make_option("--conflict", dest
="conflict_behavior",
1517 choices
=self
.conflict_behavior_choices
),
1518 optparse
.make_option("--branch", dest
="branch"),
1519 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1520 help="Shelve instead of submit. Shelved files are reverted, "
1521 "restoring the workspace to the state before the shelve"),
1522 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="append", type="int",
1523 metavar
="CHANGELIST",
1524 help="update an existing shelved changelist, implies --shelve, "
1525 "repeat in-order for multiple shelved changelists"),
1526 optparse
.make_option("--commit", dest
="commit", metavar
="COMMIT",
1527 help="submit only the specified commit(s), one commit or xxx..xxx"),
1528 optparse
.make_option("--disable-rebase", dest
="disable_rebase", action
="store_true",
1529 help="Disable rebase after submit is completed. Can be useful if you "
1530 "work from a local git branch that is not master"),
1531 optparse
.make_option("--disable-p4sync", dest
="disable_p4sync", action
="store_true",
1532 help="Skip Perforce sync of p4/master after submit or shelve"),
1533 optparse
.make_option("--no-verify", dest
="no_verify", action
="store_true",
1534 help="Bypass p4-pre-submit and p4-changelist hooks"),
1536 self
.description
= """Submit changes from git to the perforce depot.\n
1537 The `p4-pre-submit` hook is executed if it exists and is executable. It
1538 can be bypassed with the `--no-verify` command line option. The hook takes
1539 no parameters and nothing from standard input. Exiting with a non-zero status
1540 from this script prevents `git-p4 submit` from launching.
1542 One usage scenario is to run unit tests in the hook.
1544 The `p4-prepare-changelist` hook is executed right after preparing the default
1545 changelist message and before the editor is started. It takes one parameter,
1546 the name of the file that contains the changelist text. Exiting with a non-zero
1547 status from the script will abort the process.
1549 The purpose of the hook is to edit the message file in place, and it is not
1550 supressed by the `--no-verify` option. This hook is called even if
1551 `--prepare-p4-only` is set.
1553 The `p4-changelist` hook is executed after the changelist message has been
1554 edited by the user. It can be bypassed with the `--no-verify` option. It
1555 takes a single parameter, the name of the file that holds the proposed
1556 changelist text. Exiting with a non-zero status causes the command to abort.
1558 The hook is allowed to edit the changelist file and can be used to normalize
1559 the text into some project standard format. It can also be used to refuse the
1560 Submit after inspect the message file.
1562 The `p4-post-changelist` hook is invoked after the submit has successfully
1563 occurred in P4. It takes no parameters and is meant primarily for notification
1564 and cannot affect the outcome of the git p4 submit action.
1567 self
.usage
+= " [name of git branch to submit into perforce depot]"
1569 self
.detectRenames
= False
1570 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1571 self
.dry_run
= False
1573 self
.update_shelve
= list()
1575 self
.disable_rebase
= gitConfigBool("git-p4.disableRebase")
1576 self
.disable_p4sync
= gitConfigBool("git-p4.disableP4Sync")
1577 self
.prepare_p4_only
= False
1578 self
.conflict_behavior
= None
1579 self
.isWindows
= (platform
.system() == "Windows")
1580 self
.exportLabels
= False
1581 self
.p4HasMoveCommand
= p4_has_move_command()
1583 self
.no_verify
= False
1585 if gitConfig('git-p4.largeFileSystem'):
1586 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1589 if len(p4CmdList(["opened", "..."])) > 0:
1590 die("You have files opened with perforce! Close them before starting the sync.")
1592 def separate_jobs_from_description(self
, message
):
1593 """Extract and return a possible Jobs field in the commit
1594 message. It goes into a separate section in the p4 change
1597 A jobs line starts with "Jobs:" and looks like a new field
1598 in a form. Values are white-space separated on the same
1599 line or on following lines that start with a tab.
1601 This does not parse and extract the full git commit message
1602 like a p4 form. It just sees the Jobs: line as a marker
1603 to pass everything from then on directly into the p4 form,
1604 but outside the description section.
1606 Return a tuple (stripped log message, jobs string)."""
1608 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1610 return (message
, None)
1612 jobtext
= message
[m
.start():]
1613 stripped_message
= message
[:m
.start()].rstrip()
1614 return (stripped_message
, jobtext
)
1616 def prepareLogMessage(self
, template
, message
, jobs
):
1617 """Edits the template returned from "p4 change -o" to insert
1618 the message in the Description field, and the jobs text in
1622 inDescriptionSection
= False
1624 for line
in template
.split("\n"):
1625 if line
.startswith("#"):
1626 result
+= line
+ "\n"
1629 if inDescriptionSection
:
1630 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1631 inDescriptionSection
= False
1632 # insert Jobs section
1634 result
+= jobs
+ "\n"
1638 if line
.startswith("Description:"):
1639 inDescriptionSection
= True
1641 for messageLine
in message
.split("\n"):
1642 line
+= "\t" + messageLine
+ "\n"
1644 result
+= line
+ "\n"
1648 def patchRCSKeywords(self
, file, regexp
):
1649 # Attempt to zap the RCS keywords in a p4 controlled file matching the given regex
1650 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1652 with os
.fdopen(handle
, "wb") as outFile
, open(file, "rb") as inFile
:
1653 for line
in inFile
.readlines():
1654 outFile
.write(regexp
.sub(br
'$\1$', line
))
1655 # Forcibly overwrite the original file
1657 shutil
.move(outFileName
, file)
1659 # cleanup our temporary file
1660 os
.unlink(outFileName
)
1661 print("Failed to strip RCS keywords in %s" % file)
1664 print("Patched up RCS keywords in %s" % file)
1666 def p4UserForCommit(self
,id):
1667 # Return the tuple (perforce user,git email) for a given git commit id
1668 self
.getUserMapFromPerforceServer()
1669 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1670 "--format=%ae", id])
1671 gitEmail
= gitEmail
.strip()
1672 if gitEmail
not in self
.emails
:
1673 return (None,gitEmail
)
1675 return (self
.emails
[gitEmail
],gitEmail
)
1677 def checkValidP4Users(self
,commits
):
1678 # check if any git authors cannot be mapped to p4 users
1680 (user
,email
) = self
.p4UserForCommit(id)
1682 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1683 if gitConfigBool("git-p4.allowMissingP4Users"):
1686 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1688 def lastP4Changelist(self
):
1689 # Get back the last changelist number submitted in this client spec. This
1690 # then gets used to patch up the username in the change. If the same
1691 # client spec is being used by multiple processes then this might go
1693 results
= p4CmdList(["client", "-o"]) # find the current client
1697 client
= r
['Client']
1700 die("could not get client spec")
1701 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1705 die("Could not get changelist number for last submit - cannot patch up user details")
1707 def modifyChangelistUser(self
, changelist
, newUser
):
1708 # fixup the user field of a changelist after it has been submitted.
1709 changes
= p4CmdList(["change", "-o", changelist
])
1710 if len(changes
) != 1:
1711 die("Bad output from p4 change modifying %s to user %s" %
1712 (changelist
, newUser
))
1715 if c
['User'] == newUser
: return # nothing to do
1717 # p4 does not understand format version 3 and above
1718 input = marshal
.dumps(c
, 2)
1720 result
= p4CmdList(["change", "-f", "-i"], stdin
=input)
1723 if r
['code'] == 'error':
1724 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1726 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1728 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1730 def canChangeChangelists(self
):
1731 # check to see if we have p4 admin or super-user permissions, either of
1732 # which are required to modify changelists.
1733 results
= p4CmdList(["protects", self
.depotPath
])
1736 if r
['perm'] == 'admin':
1738 if r
['perm'] == 'super':
1742 def prepareSubmitTemplate(self
, changelist
=None):
1743 """Run "p4 change -o" to grab a change specification template.
1744 This does not use "p4 -G", as it is nice to keep the submission
1745 template in original order, since a human might edit it.
1747 Remove lines in the Files section that show changes to files
1748 outside the depot path we're committing into."""
1750 [upstream
, settings
] = findUpstreamBranchPoint()
1753 # A Perforce Change Specification.
1755 # Change: The change number. 'new' on a new changelist.
1756 # Date: The date this specification was last modified.
1757 # Client: The client on which the changelist was created. Read-only.
1758 # User: The user who created the changelist.
1759 # Status: Either 'pending' or 'submitted'. Read-only.
1760 # Type: Either 'public' or 'restricted'. Default is 'public'.
1761 # Description: Comments about the changelist. Required.
1762 # Jobs: What opened jobs are to be closed by this changelist.
1763 # You may delete jobs from this list. (New changelists only.)
1764 # Files: What opened files from the default changelist are to be added
1765 # to this changelist. You may delete files from this list.
1766 # (New changelists only.)
1769 inFilesSection
= False
1771 args
= ['change', '-o']
1773 args
.append(str(changelist
))
1774 for entry
in p4CmdList(args
):
1775 if 'code' not in entry
:
1777 if entry
['code'] == 'stat':
1778 change_entry
= entry
1780 if not change_entry
:
1781 die('Failed to decode output of p4 change -o')
1782 for key
, value
in change_entry
.items():
1783 if key
.startswith('File'):
1784 if 'depot-paths' in settings
:
1785 if not [p
for p
in settings
['depot-paths']
1786 if p4PathStartsWith(value
, p
)]:
1789 if not p4PathStartsWith(value
, self
.depotPath
):
1791 files_list
.append(value
)
1793 # Output in the order expected by prepareLogMessage
1794 for key
in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1795 if key
not in change_entry
:
1798 template
+= key
+ ':'
1799 if key
== 'Description':
1801 for field_line
in change_entry
[key
].splitlines():
1802 template
+= '\t'+field_line
+'\n'
1803 if len(files_list
) > 0:
1805 template
+= 'Files:\n'
1806 for path
in files_list
:
1807 template
+= '\t'+path
+'\n'
1810 def edit_template(self
, template_file
):
1811 """Invoke the editor to let the user change the submission
1812 message. Return true if okay to continue with the submit."""
1814 # if configured to skip the editing part, just submit
1815 if gitConfigBool("git-p4.skipSubmitEdit"):
1818 # look at the modification time, to check later if the user saved
1820 mtime
= os
.stat(template_file
).st_mtime
1823 if "P4EDITOR" in os
.environ
and (os
.environ
.get("P4EDITOR") != ""):
1824 editor
= os
.environ
.get("P4EDITOR")
1826 editor
= read_pipe(["git", "var", "GIT_EDITOR"]).strip()
1827 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1829 # If the file was not saved, prompt to see if this patch should
1830 # be skipped. But skip this verification step if configured so.
1831 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1834 # modification time updated means user saved the file
1835 if os
.stat(template_file
).st_mtime
> mtime
:
1838 response
= prompt("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1844 def get_diff_description(self
, editedFiles
, filesToAdd
, symlinks
):
1846 if "P4DIFF" in os
.environ
:
1847 del(os
.environ
["P4DIFF"])
1849 for editedFile
in editedFiles
:
1850 diff
+= p4_read_pipe(['diff', '-du',
1851 wildcard_encode(editedFile
)])
1855 for newFile
in filesToAdd
:
1856 newdiff
+= "==== new file ====\n"
1857 newdiff
+= "--- /dev/null\n"
1858 newdiff
+= "+++ %s\n" % newFile
1860 is_link
= os
.path
.islink(newFile
)
1861 expect_link
= newFile
in symlinks
1863 if is_link
and expect_link
:
1864 newdiff
+= "+%s\n" % os
.readlink(newFile
)
1866 f
= open(newFile
, "r")
1868 for line
in f
.readlines():
1869 newdiff
+= "+" + line
1870 except UnicodeDecodeError:
1871 pass # Found non-text data and skip, since diff description should only include text
1874 return (diff
+ newdiff
).replace('\r\n', '\n')
1876 def applyCommit(self
, id):
1877 """Apply one commit, return True if it succeeded."""
1879 print("Applying", read_pipe(["git", "show", "-s",
1880 "--format=format:%h %s", id]))
1882 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1884 diff
= read_pipe_lines(
1885 ["git", "diff-tree", "-r"] + self
.diffOpts
+ ["{}^".format(id), id])
1887 filesToChangeType
= set()
1888 filesToDelete
= set()
1890 pureRenameCopy
= set()
1892 filesToChangeExecBit
= {}
1896 diff
= parseDiffTreeEntry(line
)
1897 modifier
= diff
['status']
1899 all_files
.append(path
)
1903 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1904 filesToChangeExecBit
[path
] = diff
['dst_mode']
1905 editedFiles
.add(path
)
1906 elif modifier
== "A":
1907 filesToAdd
.add(path
)
1908 filesToChangeExecBit
[path
] = diff
['dst_mode']
1909 if path
in filesToDelete
:
1910 filesToDelete
.remove(path
)
1912 dst_mode
= int(diff
['dst_mode'], 8)
1913 if dst_mode
== 0o120000:
1916 elif modifier
== "D":
1917 filesToDelete
.add(path
)
1918 if path
in filesToAdd
:
1919 filesToAdd
.remove(path
)
1920 elif modifier
== "C":
1921 src
, dest
= diff
['src'], diff
['dst']
1922 all_files
.append(dest
)
1923 p4_integrate(src
, dest
)
1924 pureRenameCopy
.add(dest
)
1925 if diff
['src_sha1'] != diff
['dst_sha1']:
1927 pureRenameCopy
.discard(dest
)
1928 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1930 pureRenameCopy
.discard(dest
)
1931 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1933 # turn off read-only attribute
1934 os
.chmod(dest
, stat
.S_IWRITE
)
1936 editedFiles
.add(dest
)
1937 elif modifier
== "R":
1938 src
, dest
= diff
['src'], diff
['dst']
1939 all_files
.append(dest
)
1940 if self
.p4HasMoveCommand
:
1941 p4_edit(src
) # src must be open before move
1942 p4_move(src
, dest
) # opens for (move/delete, move/add)
1944 p4_integrate(src
, dest
)
1945 if diff
['src_sha1'] != diff
['dst_sha1']:
1948 pureRenameCopy
.add(dest
)
1949 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1950 if not self
.p4HasMoveCommand
:
1951 p4_edit(dest
) # with move: already open, writable
1952 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1953 if not self
.p4HasMoveCommand
:
1955 os
.chmod(dest
, stat
.S_IWRITE
)
1957 filesToDelete
.add(src
)
1958 editedFiles
.add(dest
)
1959 elif modifier
== "T":
1960 filesToChangeType
.add(path
)
1962 die("unknown modifier %s for %s" % (modifier
, path
))
1964 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1965 patchcmd
= diffcmd
+ " | git apply "
1966 tryPatchCmd
= patchcmd
+ "--check -"
1967 applyPatchCmd
= patchcmd
+ "--check --apply -"
1968 patch_succeeded
= True
1971 print("TryPatch: %s" % tryPatchCmd
)
1973 if os
.system(tryPatchCmd
) != 0:
1974 fixed_rcs_keywords
= False
1975 patch_succeeded
= False
1976 print("Unfortunately applying the change failed!")
1978 # Patch failed, maybe it's just RCS keyword woes. Look through
1979 # the patch to see if that's possible.
1980 if gitConfigBool("git-p4.attemptRCSCleanup"):
1983 for file in editedFiles | filesToDelete
:
1984 # did this file's delta contain RCS keywords?
1985 regexp
= p4_keywords_regexp_for_file(file)
1987 # this file is a possibility...look for RCS keywords.
1988 for line
in read_pipe_lines(
1989 ["git", "diff", "%s^..%s" % (id, id), file],
1991 if regexp
.search(line
):
1993 print("got keyword match on %s in %s in %s" % (regex
.pattern
, line
, file))
1994 kwfiles
[file] = regexp
1997 for file, regexp
in kwfiles
.items():
1999 print("zapping %s with %s" % (line
, regexp
.pattern
))
2000 # File is being deleted, so not open in p4. Must
2001 # disable the read-only bit on windows.
2002 if self
.isWindows
and file not in editedFiles
:
2003 os
.chmod(file, stat
.S_IWRITE
)
2004 self
.patchRCSKeywords(file, kwfiles
[file])
2005 fixed_rcs_keywords
= True
2007 if fixed_rcs_keywords
:
2008 print("Retrying the patch with RCS keywords cleaned up")
2009 if os
.system(tryPatchCmd
) == 0:
2010 patch_succeeded
= True
2011 print("Patch succeesed this time with RCS keywords cleaned")
2013 if not patch_succeeded
:
2014 for f
in editedFiles
:
2019 # Apply the patch for real, and do add/delete/+x handling.
2021 system(applyPatchCmd
, shell
=True)
2023 for f
in filesToChangeType
:
2024 p4_edit(f
, "-t", "auto")
2025 for f
in filesToAdd
:
2027 for f
in filesToDelete
:
2031 # Set/clear executable bits
2032 for f
in filesToChangeExecBit
.keys():
2033 mode
= filesToChangeExecBit
[f
]
2034 setP4ExecBit(f
, mode
)
2037 if len(self
.update_shelve
) > 0:
2038 update_shelve
= self
.update_shelve
.pop(0)
2039 p4_reopen_in_change(update_shelve
, all_files
)
2042 # Build p4 change description, starting with the contents
2043 # of the git commit message.
2045 logMessage
= extractLogMessageFromGitCommit(id)
2046 logMessage
= logMessage
.strip()
2047 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
2049 template
= self
.prepareSubmitTemplate(update_shelve
)
2050 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
2052 if self
.preserveUser
:
2053 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
2055 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
2056 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
2057 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
2058 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
2060 separatorLine
= "######## everything below this line is just the diff #######\n"
2061 if not self
.prepare_p4_only
:
2062 submitTemplate
+= separatorLine
2063 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
, symlinks
)
2065 (handle
, fileName
) = tempfile
.mkstemp()
2066 tmpFile
= os
.fdopen(handle
, "w+b")
2068 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
2069 tmpFile
.write(encode_text_stream(submitTemplate
))
2075 # Allow the hook to edit the changelist text before presenting it
2077 if not run_git_hook("p4-prepare-changelist", [fileName
]):
2080 if self
.prepare_p4_only
:
2082 # Leave the p4 tree prepared, and the submit template around
2083 # and let the user decide what to do next
2087 print("P4 workspace prepared for submission.")
2088 print("To submit or revert, go to client workspace")
2089 print(" " + self
.clientPath
)
2091 print("To submit, use \"p4 submit\" to write a new description,")
2092 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2093 " \"git p4\"." % fileName
)
2094 print("You can delete the file \"%s\" when finished." % fileName
)
2096 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
2097 print("To preserve change ownership by user %s, you must\n" \
2098 "do \"p4 change -f <change>\" after submitting and\n" \
2099 "edit the User field.")
2101 print("After submitting, renamed files must be re-synced.")
2102 print("Invoke \"p4 sync -f\" on each of these files:")
2103 for f
in pureRenameCopy
:
2107 print("To revert the changes, use \"p4 revert ...\", and delete")
2108 print("the submit template file \"%s\"" % fileName
)
2110 print("Since the commit adds new files, they must be deleted:")
2111 for f
in filesToAdd
:
2117 if self
.edit_template(fileName
):
2118 if not self
.no_verify
:
2119 if not run_git_hook("p4-changelist", [fileName
]):
2120 print("The p4-changelist hook failed.")
2124 # read the edited message and submit
2125 tmpFile
= open(fileName
, "rb")
2126 message
= decode_text_stream(tmpFile
.read())
2129 message
= message
.replace("\r\n", "\n")
2130 if message
.find(separatorLine
) != -1:
2131 submitTemplate
= message
[:message
.index(separatorLine
)]
2133 submitTemplate
= message
2135 if len(submitTemplate
.strip()) == 0:
2136 print("Changelist is empty, aborting this changelist.")
2141 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
2143 p4_write_pipe(['shelve', '-i'], submitTemplate
)
2145 p4_write_pipe(['submit', '-i'], submitTemplate
)
2146 # The rename/copy happened by applying a patch that created a
2147 # new file. This leaves it writable, which confuses p4.
2148 for f
in pureRenameCopy
:
2151 if self
.preserveUser
:
2153 # Get last changelist number. Cannot easily get it from
2154 # the submit command output as the output is
2156 changelist
= self
.lastP4Changelist()
2157 self
.modifyChangelistUser(changelist
, p4User
)
2161 run_git_hook("p4-post-changelist")
2163 # Revert changes if we skip this patch
2164 if not submitted
or self
.shelve
:
2166 print ("Reverting shelved files.")
2168 print ("Submission cancelled, undoing p4 changes.")
2170 for f
in editedFiles | filesToDelete
:
2172 for f
in filesToAdd
:
2176 if not self
.prepare_p4_only
:
2180 # Export git tags as p4 labels. Create a p4 label and then tag
2182 def exportGitTags(self
, gitTags
):
2183 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
2184 if len(validLabelRegexp
) == 0:
2185 validLabelRegexp
= defaultLabelRegexp
2186 m
= re
.compile(validLabelRegexp
)
2188 for name
in gitTags
:
2190 if not m
.match(name
):
2192 print("tag %s does not match regexp %s" % (name
, validLabelRegexp
))
2195 # Get the p4 commit this corresponds to
2196 logMessage
= extractLogMessageFromGitCommit(name
)
2197 values
= extractSettingsGitLog(logMessage
)
2199 if 'change' not in values
:
2200 # a tag pointing to something not sent to p4; ignore
2202 print("git tag %s does not give a p4 commit" % name
)
2205 changelist
= values
['change']
2207 # Get the tag details.
2211 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
2214 if re
.match(r
'tag\s+', l
):
2216 elif re
.match(r
'\s*$', l
):
2223 body
= ["lightweight tag imported by git p4\n"]
2225 # Create the label - use the same view as the client spec we are using
2226 clientSpec
= getClientSpec()
2228 labelTemplate
= "Label: %s\n" % name
2229 labelTemplate
+= "Description:\n"
2231 labelTemplate
+= "\t" + b
+ "\n"
2232 labelTemplate
+= "View:\n"
2233 for depot_side
in clientSpec
.mappings
:
2234 labelTemplate
+= "\t%s\n" % depot_side
2237 print("Would create p4 label %s for tag" % name
)
2238 elif self
.prepare_p4_only
:
2239 print("Not creating p4 label %s for tag due to option" \
2240 " --prepare-p4-only" % name
)
2242 p4_write_pipe(["label", "-i"], labelTemplate
)
2245 p4_system(["tag", "-l", name
] +
2246 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
2249 print("created p4 label for tag %s" % name
)
2251 def run(self
, args
):
2253 self
.master
= currentGitBranch()
2254 elif len(args
) == 1:
2255 self
.master
= args
[0]
2256 if not branchExists(self
.master
):
2257 die("Branch %s does not exist" % self
.master
)
2261 for i
in self
.update_shelve
:
2263 sys
.exit("invalid changelist %d" % i
)
2266 allowSubmit
= gitConfig("git-p4.allowSubmit")
2267 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
2268 die("%s is not in git-p4.allowSubmit" % self
.master
)
2270 [upstream
, settings
] = findUpstreamBranchPoint()
2271 self
.depotPath
= settings
['depot-paths'][0]
2272 if len(self
.origin
) == 0:
2273 self
.origin
= upstream
2275 if len(self
.update_shelve
) > 0:
2278 if self
.preserveUser
:
2279 if not self
.canChangeChangelists():
2280 die("Cannot preserve user names without p4 super-user or admin permissions")
2282 # if not set from the command line, try the config file
2283 if self
.conflict_behavior
is None:
2284 val
= gitConfig("git-p4.conflict")
2286 if val
not in self
.conflict_behavior_choices
:
2287 die("Invalid value '%s' for config git-p4.conflict" % val
)
2290 self
.conflict_behavior
= val
2293 print("Origin branch is " + self
.origin
)
2295 if len(self
.depotPath
) == 0:
2296 print("Internal error: cannot locate perforce depot path from existing branches")
2299 self
.useClientSpec
= False
2300 if gitConfigBool("git-p4.useclientspec"):
2301 self
.useClientSpec
= True
2302 if self
.useClientSpec
:
2303 self
.clientSpecDirs
= getClientSpec()
2305 # Check for the existence of P4 branches
2306 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
2308 if self
.useClientSpec
and not branchesDetected
:
2309 # all files are relative to the client spec
2310 self
.clientPath
= getClientRoot()
2312 self
.clientPath
= p4Where(self
.depotPath
)
2314 if self
.clientPath
== "":
2315 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
2317 print("Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
))
2318 self
.oldWorkingDirectory
= os
.getcwd()
2320 # ensure the clientPath exists
2321 new_client_dir
= False
2322 if not os
.path
.exists(self
.clientPath
):
2323 new_client_dir
= True
2324 os
.makedirs(self
.clientPath
)
2326 chdir(self
.clientPath
, is_client_path
=True)
2328 print("Would synchronize p4 checkout in %s" % self
.clientPath
)
2330 print("Synchronizing p4 checkout...")
2332 # old one was destroyed, and maybe nobody told p4
2333 p4_sync("...", "-f")
2340 committish
= self
.master
2344 if self
.commit
!= "":
2345 if self
.commit
.find("..") != -1:
2346 limits_ish
= self
.commit
.split("..")
2347 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish
[0], limits_ish
[1])]):
2348 commits
.append(line
.strip())
2351 commits
.append(self
.commit
)
2353 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, committish
)]):
2354 commits
.append(line
.strip())
2357 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2358 self
.checkAuthorship
= False
2360 self
.checkAuthorship
= True
2362 if self
.preserveUser
:
2363 self
.checkValidP4Users(commits
)
2366 # Build up a set of options to be passed to diff when
2367 # submitting each commit to p4.
2369 if self
.detectRenames
:
2370 # command-line -M arg
2371 self
.diffOpts
= ["-M"]
2373 # If not explicitly set check the config variable
2374 detectRenames
= gitConfig("git-p4.detectRenames")
2376 if detectRenames
.lower() == "false" or detectRenames
== "":
2378 elif detectRenames
.lower() == "true":
2379 self
.diffOpts
= ["-M"]
2381 self
.diffOpts
= ["-M{}".format(detectRenames
)]
2383 # no command-line arg for -C or --find-copies-harder, just
2385 detectCopies
= gitConfig("git-p4.detectCopies")
2386 if detectCopies
.lower() == "false" or detectCopies
== "":
2388 elif detectCopies
.lower() == "true":
2389 self
.diffOpts
.append("-C")
2391 self
.diffOpts
.append("-C{}".format(detectCopies
))
2393 if gitConfigBool("git-p4.detectCopiesHarder"):
2394 self
.diffOpts
.append("--find-copies-harder")
2396 num_shelves
= len(self
.update_shelve
)
2397 if num_shelves
> 0 and num_shelves
!= len(commits
):
2398 sys
.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2399 (len(commits
), num_shelves
))
2401 if not self
.no_verify
:
2403 if not run_git_hook("p4-pre-submit"):
2404 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nYou can skip " \
2405 "this pre-submission check by adding\nthe command line option '--no-verify', " \
2406 "however,\nthis will also skip the p4-changelist hook as well.")
2408 except Exception as e
:
2409 print("\nThe p4-pre-submit hook failed, aborting the submit.\n\nThe hook failed "\
2410 "with the error '{0}'".format(e
.message
) )
2414 # Apply the commits, one at a time. On failure, ask if should
2415 # continue to try the rest of the patches, or quit.
2418 print("Would apply")
2420 last
= len(commits
) - 1
2421 for i
, commit
in enumerate(commits
):
2423 print(" ", read_pipe(["git", "show", "-s",
2424 "--format=format:%h %s", commit
]))
2427 ok
= self
.applyCommit(commit
)
2429 applied
.append(commit
)
2430 if self
.prepare_p4_only
:
2432 print("Processing only the first commit due to option" \
2433 " --prepare-p4-only")
2437 # prompt for what to do, or use the option/variable
2438 if self
.conflict_behavior
== "ask":
2439 print("What do you want to do?")
2440 response
= prompt("[s]kip this commit but apply the rest, or [q]uit? ")
2441 elif self
.conflict_behavior
== "skip":
2443 elif self
.conflict_behavior
== "quit":
2446 die("Unknown conflict_behavior '%s'" %
2447 self
.conflict_behavior
)
2450 print("Skipping this commit, but applying the rest")
2455 chdir(self
.oldWorkingDirectory
)
2456 shelved_applied
= "shelved" if self
.shelve
else "applied"
2459 elif self
.prepare_p4_only
:
2461 elif len(commits
) == len(applied
):
2462 print("All commits {0}!".format(shelved_applied
))
2466 sync
.branch
= self
.branch
2467 if self
.disable_p4sync
:
2468 sync
.sync_origin_only()
2472 if not self
.disable_rebase
:
2477 if len(applied
) == 0:
2478 print("No commits {0}.".format(shelved_applied
))
2480 print("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2486 print(star
, read_pipe(["git", "show", "-s",
2487 "--format=format:%h %s", c
]))
2488 print("You will have to do 'git p4 sync' and rebase.")
2490 if gitConfigBool("git-p4.exportLabels"):
2491 self
.exportLabels
= True
2493 if self
.exportLabels
:
2494 p4Labels
= getP4Labels(self
.depotPath
)
2495 gitTags
= getGitTags()
2497 missingGitTags
= gitTags
- p4Labels
2498 self
.exportGitTags(missingGitTags
)
2500 # exit with error unless everything applied perfectly
2501 if len(commits
) != len(applied
):
2507 """Represent a p4 view ("p4 help views"), and map files in a
2508 repo according to the view."""
2510 def __init__(self
, client_name
):
2512 self
.client_prefix
= "//%s/" % client_name
2513 # cache results of "p4 where" to lookup client file locations
2514 self
.client_spec_path_cache
= {}
2516 def append(self
, view_line
):
2517 """Parse a view line, splitting it into depot and client
2518 sides. Append to self.mappings, preserving order. This
2519 is only needed for tag creation."""
2521 # Split the view line into exactly two words. P4 enforces
2522 # structure on these lines that simplifies this quite a bit.
2524 # Either or both words may be double-quoted.
2525 # Single quotes do not matter.
2526 # Double-quote marks cannot occur inside the words.
2527 # A + or - prefix is also inside the quotes.
2528 # There are no quotes unless they contain a space.
2529 # The line is already white-space stripped.
2530 # The two words are separated by a single space.
2532 if view_line
[0] == '"':
2533 # First word is double quoted. Find its end.
2534 close_quote_index
= view_line
.find('"', 1)
2535 if close_quote_index
<= 0:
2536 die("No first-word closing quote found: %s" % view_line
)
2537 depot_side
= view_line
[1:close_quote_index
]
2538 # skip closing quote and space
2539 rhs_index
= close_quote_index
+ 1 + 1
2541 space_index
= view_line
.find(" ")
2542 if space_index
<= 0:
2543 die("No word-splitting space found: %s" % view_line
)
2544 depot_side
= view_line
[0:space_index
]
2545 rhs_index
= space_index
+ 1
2547 # prefix + means overlay on previous mapping
2548 if depot_side
.startswith("+"):
2549 depot_side
= depot_side
[1:]
2551 # prefix - means exclude this path, leave out of mappings
2553 if depot_side
.startswith("-"):
2555 depot_side
= depot_side
[1:]
2558 self
.mappings
.append(depot_side
)
2560 def convert_client_path(self
, clientFile
):
2561 # chop off //client/ part to make it relative
2562 if not decode_path(clientFile
).startswith(self
.client_prefix
):
2563 die("No prefix '%s' on clientFile '%s'" %
2564 (self
.client_prefix
, clientFile
))
2565 return clientFile
[len(self
.client_prefix
):]
2567 def update_client_spec_path_cache(self
, files
):
2568 """ Caching file paths by "p4 where" batch query """
2570 # List depot file paths exclude that already cached
2571 fileArgs
= [f
['path'] for f
in files
if decode_path(f
['path']) not in self
.client_spec_path_cache
]
2573 if len(fileArgs
) == 0:
2574 return # All files in cache
2576 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2577 for res
in where_result
:
2578 if "code" in res
and res
["code"] == "error":
2579 # assume error is "... file(s) not in client view"
2581 if "clientFile" not in res
:
2582 die("No clientFile in 'p4 where' output")
2584 # it will list all of them, but only one not unmap-ped
2586 depot_path
= decode_path(res
['depotFile'])
2587 if gitConfigBool("core.ignorecase"):
2588 depot_path
= depot_path
.lower()
2589 self
.client_spec_path_cache
[depot_path
] = self
.convert_client_path(res
["clientFile"])
2591 # not found files or unmap files set to ""
2592 for depotFile
in fileArgs
:
2593 depotFile
= decode_path(depotFile
)
2594 if gitConfigBool("core.ignorecase"):
2595 depotFile
= depotFile
.lower()
2596 if depotFile
not in self
.client_spec_path_cache
:
2597 self
.client_spec_path_cache
[depotFile
] = b
''
2599 def map_in_client(self
, depot_path
):
2600 """Return the relative location in the client where this
2601 depot file should live. Returns "" if the file should
2602 not be mapped in the client."""
2604 if gitConfigBool("core.ignorecase"):
2605 depot_path
= depot_path
.lower()
2607 if depot_path
in self
.client_spec_path_cache
:
2608 return self
.client_spec_path_cache
[depot_path
]
2610 die( "Error: %s is not found in client spec path" % depot_path
)
2613 def cloneExcludeCallback(option
, opt_str
, value
, parser
):
2614 # prepend "/" because the first "/" was consumed as part of the option itself.
2615 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2616 parser
.values
.cloneExclude
+= ["/" + re
.sub(r
"\.\.\.$", "", value
)]
2618 class P4Sync(Command
, P4UserMap
):
2621 Command
.__init
__(self
)
2622 P4UserMap
.__init
__(self
)
2624 optparse
.make_option("--branch", dest
="branch"),
2625 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2626 optparse
.make_option("--changesfile", dest
="changesFile"),
2627 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2628 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2629 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2630 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2631 help="Import into refs/heads/ , not refs/remotes"),
2632 optparse
.make_option("--max-changes", dest
="maxChanges",
2633 help="Maximum number of changes to import"),
2634 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2635 help="Internal block size to use when iteratively calling p4 changes"),
2636 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2637 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2638 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2639 help="Only sync files that are included in the Perforce Client Spec"),
2640 optparse
.make_option("-/", dest
="cloneExclude",
2641 action
="callback", callback
=cloneExcludeCallback
, type="string",
2642 help="exclude depot path"),
2644 self
.description
= """Imports from Perforce into a git repository.\n
2646 //depot/my/project/ -- to import the current head
2647 //depot/my/project/@all -- to import everything
2648 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2650 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2652 self
.usage
+= " //depot/path[@revRange]"
2654 self
.createdBranches
= set()
2655 self
.committedChanges
= set()
2657 self
.detectBranches
= False
2658 self
.detectLabels
= False
2659 self
.importLabels
= False
2660 self
.changesFile
= ""
2661 self
.syncWithOrigin
= True
2662 self
.importIntoRemotes
= True
2663 self
.maxChanges
= ""
2664 self
.changes_block_size
= None
2665 self
.keepRepoPath
= False
2666 self
.depotPaths
= None
2667 self
.p4BranchesInGit
= []
2668 self
.cloneExclude
= []
2669 self
.useClientSpec
= False
2670 self
.useClientSpec_from_options
= False
2671 self
.clientSpecDirs
= None
2672 self
.tempBranches
= []
2673 self
.tempBranchLocation
= "refs/git-p4-tmp"
2674 self
.largeFileSystem
= None
2675 self
.suppress_meta_comment
= False
2677 if gitConfig('git-p4.largeFileSystem'):
2678 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2679 self
.largeFileSystem
= largeFileSystemConstructor(
2680 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2683 if gitConfig("git-p4.syncFromOrigin") == "false":
2684 self
.syncWithOrigin
= False
2686 self
.depotPaths
= []
2687 self
.changeRange
= ""
2688 self
.previousDepotPaths
= []
2689 self
.hasOrigin
= False
2691 # map from branch depot path to parent branch
2692 self
.knownBranches
= {}
2693 self
.initialParents
= {}
2695 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2698 # Force a checkpoint in fast-import and wait for it to finish
2699 def checkpoint(self
):
2700 self
.gitStream
.write("checkpoint\n\n")
2701 self
.gitStream
.write("progress checkpoint\n\n")
2702 self
.gitStream
.flush()
2703 out
= self
.gitOutput
.readline()
2705 print("checkpoint finished: " + out
)
2707 def isPathWanted(self
, path
):
2708 for p
in self
.cloneExclude
:
2710 if p4PathStartsWith(path
, p
):
2712 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2713 elif path
.lower() == p
.lower():
2715 for p
in self
.depotPaths
:
2716 if p4PathStartsWith(path
, decode_path(p
)):
2720 def extractFilesFromCommit(self
, commit
, shelved
=False, shelved_cl
= 0):
2723 while "depotFile%s" % fnum
in commit
:
2724 path
= commit
["depotFile%s" % fnum
]
2725 found
= self
.isPathWanted(decode_path(path
))
2732 file["rev"] = commit
["rev%s" % fnum
]
2733 file["action"] = commit
["action%s" % fnum
]
2734 file["type"] = commit
["type%s" % fnum
]
2736 file["shelved_cl"] = int(shelved_cl
)
2741 def extractJobsFromCommit(self
, commit
):
2744 while "job%s" % jnum
in commit
:
2745 job
= commit
["job%s" % jnum
]
2750 def stripRepoPath(self
, path
, prefixes
):
2751 """When streaming files, this is called to map a p4 depot path
2752 to where it should go in git. The prefixes are either
2753 self.depotPaths, or self.branchPrefixes in the case of
2754 branch detection."""
2756 if self
.useClientSpec
:
2757 # branch detection moves files up a level (the branch name)
2758 # from what client spec interpretation gives
2759 path
= decode_path(self
.clientSpecDirs
.map_in_client(path
))
2760 if self
.detectBranches
:
2761 for b
in self
.knownBranches
:
2762 if p4PathStartsWith(path
, b
+ "/"):
2763 path
= path
[len(b
)+1:]
2765 elif self
.keepRepoPath
:
2766 # Preserve everything in relative path name except leading
2767 # //depot/; just look at first prefix as they all should
2768 # be in the same depot.
2769 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2770 if p4PathStartsWith(path
, depot
):
2771 path
= path
[len(depot
):]
2775 if p4PathStartsWith(path
, p
):
2776 path
= path
[len(p
):]
2779 path
= wildcard_decode(path
)
2782 def splitFilesIntoBranches(self
, commit
):
2783 """Look at each depotFile in the commit to figure out to what
2784 branch it belongs."""
2786 if self
.clientSpecDirs
:
2787 files
= self
.extractFilesFromCommit(commit
)
2788 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2792 while "depotFile%s" % fnum
in commit
:
2793 raw_path
= commit
["depotFile%s" % fnum
]
2794 path
= decode_path(raw_path
)
2795 found
= self
.isPathWanted(path
)
2801 file["path"] = raw_path
2802 file["rev"] = commit
["rev%s" % fnum
]
2803 file["action"] = commit
["action%s" % fnum
]
2804 file["type"] = commit
["type%s" % fnum
]
2807 # start with the full relative path where this file would
2809 if self
.useClientSpec
:
2810 relPath
= decode_path(self
.clientSpecDirs
.map_in_client(path
))
2812 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2814 for branch
in self
.knownBranches
.keys():
2815 # add a trailing slash so that a commit into qt/4.2foo
2816 # doesn't end up in qt/4.2, e.g.
2817 if p4PathStartsWith(relPath
, branch
+ "/"):
2818 if branch
not in branches
:
2819 branches
[branch
] = []
2820 branches
[branch
].append(file)
2825 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2826 self
.gitStream
.write(encode_text_stream(u
'M {} inline {}\n'.format(gitMode
, relPath
)))
2827 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2829 self
.gitStream
.write(d
)
2830 self
.gitStream
.write('\n')
2832 def encodeWithUTF8(self
, path
):
2834 path
.decode('ascii')
2837 if gitConfig('git-p4.pathEncoding'):
2838 encoding
= gitConfig('git-p4.pathEncoding')
2839 path
= path
.decode(encoding
, 'replace').encode('utf8', 'replace')
2841 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, path
))
2844 # output one file from the P4 stream
2845 # - helper for streamP4Files
2847 def streamOneP4File(self
, file, contents
):
2848 file_path
= file['depotFile']
2849 relPath
= self
.stripRepoPath(decode_path(file_path
), self
.branchPrefixes
)
2852 if 'fileSize' in self
.stream_file
:
2853 size
= int(self
.stream_file
['fileSize'])
2855 size
= 0 # deleted files don't get a fileSize apparently
2856 sys
.stdout
.write('\r%s --> %s (%s)\n' % (
2857 file_path
, relPath
, format_size_human_readable(size
)))
2860 (type_base
, type_mods
) = split_p4_type(file["type"])
2863 if "x" in type_mods
:
2865 if type_base
== "symlink":
2867 # p4 print on a symlink sometimes contains "target\n";
2868 # if it does, remove the newline
2869 data
= ''.join(decode_text_stream(c
) for c
in contents
)
2871 # Some version of p4 allowed creating a symlink that pointed
2872 # to nothing. This causes p4 errors when checking out such
2873 # a change, and errors here too. Work around it by ignoring
2874 # the bad symlink; hopefully a future change fixes it.
2875 print("\nIgnoring empty symlink in %s" % file_path
)
2877 elif data
[-1] == '\n':
2878 contents
= [data
[:-1]]
2882 if type_base
== "utf16":
2883 # p4 delivers different text in the python output to -G
2884 # than it does when using "print -o", or normal p4 client
2885 # operations. utf16 is converted to ascii or utf8, perhaps.
2886 # But ascii text saved as -t utf16 is completely mangled.
2887 # Invoke print -o to get the real contents.
2889 # On windows, the newlines will always be mangled by print, so put
2890 # them back too. This is not needed to the cygwin windows version,
2891 # just the native "NT" type.
2894 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw
=True)
2895 except Exception as e
:
2896 if 'Translation of file content failed' in str(e
):
2897 type_base
= 'binary'
2901 if p4_version_string().find('/NT') >= 0:
2902 text
= text
.replace(b
'\r\n', b
'\n')
2905 if type_base
== "apple":
2906 # Apple filetype files will be streamed as a concatenation of
2907 # its appledouble header and the contents. This is useless
2908 # on both macs and non-macs. If using "print -q -o xx", it
2909 # will create "xx" with the data, and "%xx" with the header.
2910 # This is also not very useful.
2912 # Ideally, someday, this script can learn how to generate
2913 # appledouble files directly and import those to git, but
2914 # non-mac machines can never find a use for apple filetype.
2915 print("\nIgnoring apple filetype file %s" % file['depotFile'])
2918 # Note that we do not try to de-mangle keywords on utf16 files,
2919 # even though in theory somebody may want that.
2920 regexp
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2922 contents
= [regexp
.sub(br
'$\1$', c
) for c
in contents
]
2924 if self
.largeFileSystem
:
2925 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2927 self
.writeToGitStream(git_mode
, relPath
, contents
)
2929 def streamOneP4Deletion(self
, file):
2930 relPath
= self
.stripRepoPath(decode_path(file['path']), self
.branchPrefixes
)
2932 sys
.stdout
.write("delete %s\n" % relPath
)
2934 self
.gitStream
.write(encode_text_stream(u
'D {}\n'.format(relPath
)))
2936 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2937 self
.largeFileSystem
.removeLargeFile(relPath
)
2939 # handle another chunk of streaming data
2940 def streamP4FilesCb(self
, marshalled
):
2942 # catch p4 errors and complain
2944 if "code" in marshalled
:
2945 if marshalled
["code"] == "error":
2946 if "data" in marshalled
:
2947 err
= marshalled
["data"].rstrip()
2949 if not err
and 'fileSize' in self
.stream_file
:
2950 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2951 if required_bytes
> 0:
2952 err
= 'Not enough space left on %s! Free at least %s.' % (
2953 os
.getcwd(), format_size_human_readable(required_bytes
))
2957 if self
.stream_have_file_info
:
2958 if "depotFile" in self
.stream_file
:
2959 f
= self
.stream_file
["depotFile"]
2960 # force a failure in fast-import, else an empty
2961 # commit will be made
2962 self
.gitStream
.write("\n")
2963 self
.gitStream
.write("die-now\n")
2964 self
.gitStream
.close()
2965 # ignore errors, but make sure it exits first
2966 self
.importProcess
.wait()
2968 die("Error from p4 print for %s: %s" % (f
, err
))
2970 die("Error from p4 print: %s" % err
)
2972 if 'depotFile' in marshalled
and self
.stream_have_file_info
:
2973 # start of a new file - output the old one first
2974 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2975 self
.stream_file
= {}
2976 self
.stream_contents
= []
2977 self
.stream_have_file_info
= False
2979 # pick up the new file information... for the
2980 # 'data' field we need to append to our array
2981 for k
in marshalled
.keys():
2983 if 'streamContentSize' not in self
.stream_file
:
2984 self
.stream_file
['streamContentSize'] = 0
2985 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2986 self
.stream_contents
.append(marshalled
['data'])
2988 self
.stream_file
[k
] = marshalled
[k
]
2991 'streamContentSize' in self
.stream_file
and
2992 'fileSize' in self
.stream_file
and
2993 'depotFile' in self
.stream_file
):
2994 size
= int(self
.stream_file
["fileSize"])
2996 progress
= 100*self
.stream_file
['streamContentSize']/size
2997 sys
.stdout
.write('\r%s %d%% (%s)' % (
2998 self
.stream_file
['depotFile'], progress
,
2999 format_size_human_readable(size
)))
3002 self
.stream_have_file_info
= True
3004 # Stream directly from "p4 files" into "git fast-import"
3005 def streamP4Files(self
, files
):
3011 filesForCommit
.append(f
)
3012 if f
['action'] in self
.delete_actions
:
3013 filesToDelete
.append(f
)
3015 filesToRead
.append(f
)
3018 for f
in filesToDelete
:
3019 self
.streamOneP4Deletion(f
)
3021 if len(filesToRead
) > 0:
3022 self
.stream_file
= {}
3023 self
.stream_contents
= []
3024 self
.stream_have_file_info
= False
3026 # curry self argument
3027 def streamP4FilesCbSelf(entry
):
3028 self
.streamP4FilesCb(entry
)
3031 for f
in filesToRead
:
3032 if 'shelved_cl' in f
:
3033 # Handle shelved CLs using the "p4 print file@=N" syntax to print
3035 fileArg
= f
['path'] + encode_text_stream('@={}'.format(f
['shelved_cl']))
3037 fileArg
= f
['path'] + encode_text_stream('#{}'.format(f
['rev']))
3039 fileArgs
.append(fileArg
)
3041 p4CmdList(["-x", "-", "print"],
3043 cb
=streamP4FilesCbSelf
)
3046 if 'depotFile' in self
.stream_file
:
3047 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
3049 def make_email(self
, userid
):
3050 if userid
in self
.users
:
3051 return self
.users
[userid
]
3053 return "%s <a@b>" % userid
3055 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
3056 """ Stream a p4 tag.
3057 commit is either a git commit, or a fast-import mark, ":<p4commit>"
3061 print("writing tag %s for commit %s" % (labelName
, commit
))
3062 gitStream
.write("tag %s\n" % labelName
)
3063 gitStream
.write("from %s\n" % commit
)
3065 if 'Owner' in labelDetails
:
3066 owner
= labelDetails
["Owner"]
3070 # Try to use the owner of the p4 label, or failing that,
3071 # the current p4 user id.
3073 email
= self
.make_email(owner
)
3075 email
= self
.make_email(self
.p4UserId())
3076 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
3078 gitStream
.write("tagger %s\n" % tagger
)
3080 print("labelDetails=",labelDetails
)
3081 if 'Description' in labelDetails
:
3082 description
= labelDetails
['Description']
3084 description
= 'Label from git p4'
3086 gitStream
.write("data %d\n" % len(description
))
3087 gitStream
.write(description
)
3088 gitStream
.write("\n")
3090 def inClientSpec(self
, path
):
3091 if not self
.clientSpecDirs
:
3093 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
3094 if not inClientSpec
and self
.verbose
:
3095 print('Ignoring file outside of client spec: {0}'.format(path
))
3098 def hasBranchPrefix(self
, path
):
3099 if not self
.branchPrefixes
:
3101 hasPrefix
= [p
for p
in self
.branchPrefixes
3102 if p4PathStartsWith(path
, p
)]
3103 if not hasPrefix
and self
.verbose
:
3104 print('Ignoring file outside of prefix: {0}'.format(path
))
3107 def findShadowedFiles(self
, files
, change
):
3108 # Perforce allows you commit files and directories with the same name,
3109 # so you could have files //depot/foo and //depot/foo/bar both checked
3110 # in. A p4 sync of a repository in this state fails. Deleting one of
3111 # the files recovers the repository.
3113 # Git will not allow the broken state to exist and only the most recent
3114 # of the conflicting names is left in the repository. When one of the
3115 # conflicting files is deleted we need to re-add the other one to make
3116 # sure the git repository recovers in the same way as perforce.
3117 deleted
= [f
for f
in files
if f
['action'] in self
.delete_actions
]
3120 path
= decode_path(f
['path'])
3121 to_check
.add(path
+ '/...')
3123 path
= path
.rsplit("/", 1)[0]
3124 if path
== "/" or path
in to_check
:
3127 to_check
= ['%s@%s' % (wildcard_encode(p
), change
) for p
in to_check
3128 if self
.hasBranchPrefix(p
)]
3130 stat_result
= p4CmdList(["-x", "-", "fstat", "-T",
3131 "depotFile,headAction,headRev,headType"], stdin
=to_check
)
3132 for record
in stat_result
:
3133 if record
['code'] != 'stat':
3135 if record
['headAction'] in self
.delete_actions
:
3139 'path': record
['depotFile'],
3140 'rev': record
['headRev'],
3141 'type': record
['headType']})
3143 def commit(self
, details
, files
, branch
, parent
= "", allow_empty
=False):
3144 epoch
= details
["time"]
3145 author
= details
["user"]
3146 jobs
= self
.extractJobsFromCommit(details
)
3149 print('commit into {0}'.format(branch
))
3151 files
= [f
for f
in files
3152 if self
.hasBranchPrefix(decode_path(f
['path']))]
3153 self
.findShadowedFiles(files
, details
['change'])
3155 if self
.clientSpecDirs
:
3156 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
3158 files
= [f
for f
in files
if self
.inClientSpec(decode_path(f
['path']))]
3160 if gitConfigBool('git-p4.keepEmptyCommits'):
3163 if not files
and not allow_empty
:
3164 print('Ignoring revision {0} as it would produce an empty commit.'
3165 .format(details
['change']))
3168 self
.gitStream
.write("commit %s\n" % branch
)
3169 self
.gitStream
.write("mark :%s\n" % details
["change"])
3170 self
.committedChanges
.add(int(details
["change"]))
3172 if author
not in self
.users
:
3173 self
.getUserMapFromPerforceServer()
3174 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
3176 self
.gitStream
.write("committer %s\n" % committer
)
3178 self
.gitStream
.write("data <<EOT\n")
3179 self
.gitStream
.write(details
["desc"])
3181 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
3183 if not self
.suppress_meta_comment
:
3184 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3185 (','.join(self
.branchPrefixes
), details
["change"]))
3186 if len(details
['options']) > 0:
3187 self
.gitStream
.write(": options = %s" % details
['options'])
3188 self
.gitStream
.write("]\n")
3190 self
.gitStream
.write("EOT\n\n")
3194 print("parent %s" % parent
)
3195 self
.gitStream
.write("from %s\n" % parent
)
3197 self
.streamP4Files(files
)
3198 self
.gitStream
.write("\n")
3200 change
= int(details
["change"])
3202 if change
in self
.labels
:
3203 label
= self
.labels
[change
]
3204 labelDetails
= label
[0]
3205 labelRevisions
= label
[1]
3207 print("Change %s is labelled %s" % (change
, labelDetails
))
3209 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
3210 for p
in self
.branchPrefixes
])
3212 if len(files
) == len(labelRevisions
):
3216 if info
["action"] in self
.delete_actions
:
3218 cleanedFiles
[info
["depotFile"]] = info
["rev"]
3220 if cleanedFiles
== labelRevisions
:
3221 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
3225 print("Tag %s does not match with change %s: files do not match."
3226 % (labelDetails
["label"], change
))
3230 print("Tag %s does not match with change %s: file count is different."
3231 % (labelDetails
["label"], change
))
3233 # Build a dictionary of changelists and labels, for "detect-labels" option.
3234 def getLabels(self
):
3237 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
3238 if len(l
) > 0 and not self
.silent
:
3239 print("Finding files belonging to labels in %s" % self
.depotPaths
)
3242 label
= output
["label"]
3246 print("Querying files for label %s" % label
)
3247 for file in p4CmdList(["files"] +
3248 ["%s...@%s" % (p
, label
)
3249 for p
in self
.depotPaths
]):
3250 revisions
[file["depotFile"]] = file["rev"]
3251 change
= int(file["change"])
3252 if change
> newestChange
:
3253 newestChange
= change
3255 self
.labels
[newestChange
] = [output
, revisions
]
3258 print("Label changes: %s" % self
.labels
.keys())
3260 # Import p4 labels as git tags. A direct mapping does not
3261 # exist, so assume that if all the files are at the same revision
3262 # then we can use that, or it's something more complicated we should
3264 def importP4Labels(self
, stream
, p4Labels
):
3266 print("import p4 labels: " + ' '.join(p4Labels
))
3268 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
3269 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
3270 if len(validLabelRegexp
) == 0:
3271 validLabelRegexp
= defaultLabelRegexp
3272 m
= re
.compile(validLabelRegexp
)
3274 for name
in p4Labels
:
3277 if not m
.match(name
):
3279 print("label %s does not match regexp %s" % (name
,validLabelRegexp
))
3282 if name
in ignoredP4Labels
:
3285 labelDetails
= p4CmdList(['label', "-o", name
])[0]
3287 # get the most recent changelist for each file in this label
3288 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
3289 for p
in self
.depotPaths
])
3291 if 'change' in change
:
3292 # find the corresponding git commit; take the oldest commit
3293 changelist
= int(change
['change'])
3294 if changelist
in self
.committedChanges
:
3295 gitCommit
= ":%d" % changelist
# use a fast-import mark
3298 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
3299 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
3300 if len(gitCommit
) == 0:
3301 print("importing label %s: could not find git commit for changelist %d" % (name
, changelist
))
3304 gitCommit
= gitCommit
.strip()
3307 # Convert from p4 time format
3309 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
3311 print("Could not convert label time %s" % labelDetails
['Update'])
3314 when
= int(time
.mktime(tmwhen
))
3315 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
3317 print("p4 label %s mapped to git commit %s" % (name
, gitCommit
))
3320 print("Label %s has no changelists - possibly deleted?" % name
)
3323 # We can't import this label; don't try again as it will get very
3324 # expensive repeatedly fetching all the files for labels that will
3325 # never be imported. If the label is moved in the future, the
3326 # ignore will need to be removed manually.
3327 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
3329 def guessProjectName(self
):
3330 for p
in self
.depotPaths
:
3333 p
= p
[p
.strip().rfind("/") + 1:]
3334 if not p
.endswith("/"):
3338 def getBranchMapping(self
):
3339 lostAndFoundBranches
= set()
3341 user
= gitConfig("git-p4.branchUser")
3343 for info
in p4CmdList(
3344 ["branches"] + (["-u", user
] if len(user
) > 0 else [])):
3345 details
= p4Cmd(["branch", "-o", info
["branch"]])
3347 while "View%s" % viewIdx
in details
:
3348 paths
= details
["View%s" % viewIdx
].split(" ")
3349 viewIdx
= viewIdx
+ 1
3350 # require standard //depot/foo/... //depot/bar/... mapping
3351 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
3354 destination
= paths
[1]
3356 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
3357 source
= source
[len(self
.depotPaths
[0]):-4]
3358 destination
= destination
[len(self
.depotPaths
[0]):-4]
3360 if destination
in self
.knownBranches
:
3362 print("p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
))
3363 print("but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
))
3366 self
.knownBranches
[destination
] = source
3368 lostAndFoundBranches
.discard(destination
)
3370 if source
not in self
.knownBranches
:
3371 lostAndFoundBranches
.add(source
)
3373 # Perforce does not strictly require branches to be defined, so we also
3374 # check git config for a branch list.
3376 # Example of branch definition in git config file:
3378 # branchList=main:branchA
3379 # branchList=main:branchB
3380 # branchList=branchA:branchC
3381 configBranches
= gitConfigList("git-p4.branchList")
3382 for branch
in configBranches
:
3384 (source
, destination
) = branch
.split(":")
3385 self
.knownBranches
[destination
] = source
3387 lostAndFoundBranches
.discard(destination
)
3389 if source
not in self
.knownBranches
:
3390 lostAndFoundBranches
.add(source
)
3393 for branch
in lostAndFoundBranches
:
3394 self
.knownBranches
[branch
] = branch
3396 def getBranchMappingFromGitBranches(self
):
3397 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3398 for branch
in branches
.keys():
3399 if branch
== "master":
3402 branch
= branch
[len(self
.projectName
):]
3403 self
.knownBranches
[branch
] = branch
3405 def updateOptionDict(self
, d
):
3407 if self
.keepRepoPath
:
3408 option_keys
['keepRepoPath'] = 1
3410 d
["options"] = ' '.join(sorted(option_keys
.keys()))
3412 def readOptions(self
, d
):
3413 self
.keepRepoPath
= ('options' in d
3414 and ('keepRepoPath' in d
['options']))
3416 def gitRefForBranch(self
, branch
):
3417 if branch
== "main":
3418 return self
.refPrefix
+ "master"
3420 if len(branch
) <= 0:
3423 return self
.refPrefix
+ self
.projectName
+ branch
3425 def gitCommitByP4Change(self
, ref
, change
):
3427 print("looking in ref " + ref
+ " for change %s using bisect..." % change
)
3430 latestCommit
= parseRevision(ref
)
3434 print("trying: earliest %s latest %s" % (earliestCommit
, latestCommit
))
3435 next
= read_pipe(["git", "rev-list", "--bisect",
3436 latestCommit
, earliestCommit
]).strip()
3441 log
= extractLogMessageFromGitCommit(next
)
3442 settings
= extractSettingsGitLog(log
)
3443 currentChange
= int(settings
['change'])
3445 print("current change %s" % currentChange
)
3447 if currentChange
== change
:
3449 print("found %s" % next
)
3452 if currentChange
< change
:
3453 earliestCommit
= "^%s" % next
3455 if next
== latestCommit
:
3456 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref
, change
))
3457 latestCommit
= "%s^@" % next
3461 def importNewBranch(self
, branch
, maxChange
):
3462 # make fast-import flush all changes to disk and update the refs using the checkpoint
3463 # command so that we can try to find the branch parent in the git history
3464 self
.gitStream
.write("checkpoint\n\n");
3465 self
.gitStream
.flush();
3466 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3467 range = "@1,%s" % maxChange
3468 #print "prefix" + branchPrefix
3469 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3470 if len(changes
) <= 0:
3472 firstChange
= changes
[0]
3473 #print "first change in branch: %s" % firstChange
3474 sourceBranch
= self
.knownBranches
[branch
]
3475 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3476 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3477 #print "source " + sourceBranch
3479 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3480 #print "branch parent: %s" % branchParentChange
3481 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3482 if len(gitParent
) > 0:
3483 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3484 #print "parent git commit: %s" % gitParent
3486 self
.importChanges(changes
)
3489 def searchParent(self
, parent
, branch
, target
):
3490 targetTree
= read_pipe(["git", "rev-parse",
3491 "{}^{{tree}}".format(target
)]).strip()
3492 for line
in read_pipe_lines(["git", "rev-list", "--format=%H %T",
3493 "--no-merges", parent
]):
3494 if line
.startswith("commit "):
3496 commit
, tree
= line
.strip().split(" ")
3497 if tree
== targetTree
:
3499 print("Found parent of %s in commit %s" % (branch
, commit
))
3503 def importChanges(self
, changes
, origin_revision
=0):
3505 for change
in changes
:
3506 description
= p4_describe(change
)
3507 self
.updateOptionDict(description
)
3510 sys
.stdout
.write("\rImporting revision %s (%d%%)" % (
3511 change
, (cnt
* 100) // len(changes
)))
3516 if self
.detectBranches
:
3517 branches
= self
.splitFilesIntoBranches(description
)
3518 for branch
in branches
.keys():
3520 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3521 self
.branchPrefixes
= [ branchPrefix
]
3525 filesForCommit
= branches
[branch
]
3528 print("branch is %s" % branch
)
3530 self
.updatedBranches
.add(branch
)
3532 if branch
not in self
.createdBranches
:
3533 self
.createdBranches
.add(branch
)
3534 parent
= self
.knownBranches
[branch
]
3535 if parent
== branch
:
3538 fullBranch
= self
.projectName
+ branch
3539 if fullBranch
not in self
.p4BranchesInGit
:
3541 print("\n Importing new branch %s" % fullBranch
);
3542 if self
.importNewBranch(branch
, change
- 1):
3544 self
.p4BranchesInGit
.append(fullBranch
)
3546 print("\n Resuming with change %s" % change
);
3549 print("parent determined through known branches: %s" % parent
)
3551 branch
= self
.gitRefForBranch(branch
)
3552 parent
= self
.gitRefForBranch(parent
)
3555 print("looking for initial parent for %s; current parent is %s" % (branch
, parent
))
3557 if len(parent
) == 0 and branch
in self
.initialParents
:
3558 parent
= self
.initialParents
[branch
]
3559 del self
.initialParents
[branch
]
3563 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3565 print("Creating temporary branch: " + tempBranch
)
3566 self
.commit(description
, filesForCommit
, tempBranch
)
3567 self
.tempBranches
.append(tempBranch
)
3569 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3571 self
.commit(description
, filesForCommit
, branch
, blob
)
3574 print("Parent of %s not found. Committing into head of %s" % (branch
, parent
))
3575 self
.commit(description
, filesForCommit
, branch
, parent
)
3577 files
= self
.extractFilesFromCommit(description
)
3578 self
.commit(description
, files
, self
.branch
,
3580 # only needed once, to connect to the previous commit
3581 self
.initialParent
= ""
3583 print(self
.gitError
.read())
3586 def sync_origin_only(self
):
3587 if self
.syncWithOrigin
:
3588 self
.hasOrigin
= originP4BranchesExist()
3591 print('Syncing with origin first, using "git fetch origin"')
3592 system(["git", "fetch", "origin"])
3594 def importHeadRevision(self
, revision
):
3595 print("Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
))
3598 details
["user"] = "git perforce import user"
3599 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3600 % (' '.join(self
.depotPaths
), revision
))
3601 details
["change"] = revision
3605 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3607 for info
in p4CmdList(["files"] + fileArgs
):
3609 if 'code' in info
and info
['code'] == 'error':
3610 sys
.stderr
.write("p4 returned an error: %s\n"
3612 if info
['data'].find("must refer to client") >= 0:
3613 sys
.stderr
.write("This particular p4 error is misleading.\n")
3614 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3615 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3617 if 'p4ExitCode' in info
:
3618 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3622 change
= int(info
["change"])
3623 if change
> newestRevision
:
3624 newestRevision
= change
3626 if info
["action"] in self
.delete_actions
:
3627 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3628 #fileCnt = fileCnt + 1
3631 for prop
in ["depotFile", "rev", "action", "type" ]:
3632 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3634 fileCnt
= fileCnt
+ 1
3636 details
["change"] = newestRevision
3638 # Use time from top-most change so that all git p4 clones of
3639 # the same p4 repo have the same commit SHA1s.
3640 res
= p4_describe(newestRevision
)
3641 details
["time"] = res
["time"]
3643 self
.updateOptionDict(details
)
3645 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3646 except IOError as err
:
3647 print("IO error with git fast-import. Is your git version recent enough?")
3648 print("IO error details: {}".format(err
))
3649 print(self
.gitError
.read())
3652 def importRevisions(self
, args
, branch_arg_given
):
3655 if len(self
.changesFile
) > 0:
3656 with
open(self
.changesFile
) as f
:
3657 output
= f
.readlines()
3660 changeSet
.add(int(line
))
3662 for change
in changeSet
:
3663 changes
.append(change
)
3667 # catch "git p4 sync" with no new branches, in a repo that
3668 # does not have any existing p4 branches
3670 if not self
.p4BranchesInGit
:
3671 raise P4CommandException("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3673 # The default branch is master, unless --branch is used to
3674 # specify something else. Make sure it exists, or complain
3675 # nicely about how to use --branch.
3676 if not self
.detectBranches
:
3677 if not branch_exists(self
.branch
):
3678 if branch_arg_given
:
3679 raise P4CommandException("Error: branch %s does not exist." % self
.branch
)
3681 raise P4CommandException("Error: no branch %s; perhaps specify one with --branch." %
3685 print("Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3687 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3689 if len(self
.maxChanges
) > 0:
3690 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3692 if len(changes
) == 0:
3694 print("No changes to import!")
3696 if not self
.silent
and not self
.detectBranches
:
3697 print("Import destination: %s" % self
.branch
)
3699 self
.updatedBranches
= set()
3701 if not self
.detectBranches
:
3703 # start a new branch
3704 self
.initialParent
= ""
3706 # build on a previous revision
3707 self
.initialParent
= parseRevision(self
.branch
)
3709 self
.importChanges(changes
)
3713 if len(self
.updatedBranches
) > 0:
3714 sys
.stdout
.write("Updated branches: ")
3715 for b
in self
.updatedBranches
:
3716 sys
.stdout
.write("%s " % b
)
3717 sys
.stdout
.write("\n")
3719 def openStreams(self
):
3720 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3721 stdin
=subprocess
.PIPE
,
3722 stdout
=subprocess
.PIPE
,
3723 stderr
=subprocess
.PIPE
);
3724 self
.gitOutput
= self
.importProcess
.stdout
3725 self
.gitStream
= self
.importProcess
.stdin
3726 self
.gitError
= self
.importProcess
.stderr
3728 if bytes
is not str:
3729 # Wrap gitStream.write() so that it can be called using `str` arguments
3730 def make_encoded_write(write
):
3731 def encoded_write(s
):
3732 return write(s
.encode() if isinstance(s
, str) else s
)
3733 return encoded_write
3735 self
.gitStream
.write
= make_encoded_write(self
.gitStream
.write
)
3737 def closeStreams(self
):
3738 if self
.gitStream
is None:
3740 self
.gitStream
.close()
3741 if self
.importProcess
.wait() != 0:
3742 die("fast-import failed: %s" % self
.gitError
.read())
3743 self
.gitOutput
.close()
3744 self
.gitError
.close()
3745 self
.gitStream
= None
3747 def run(self
, args
):
3748 if self
.importIntoRemotes
:
3749 self
.refPrefix
= "refs/remotes/p4/"
3751 self
.refPrefix
= "refs/heads/p4/"
3753 self
.sync_origin_only()
3755 branch_arg_given
= bool(self
.branch
)
3756 if len(self
.branch
) == 0:
3757 self
.branch
= self
.refPrefix
+ "master"
3758 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3759 system(["git", "update-ref", self
.branch
, "refs/heads/p4"])
3760 system(["git", "branch", "-D", "p4"])
3762 # accept either the command-line option, or the configuration variable
3763 if self
.useClientSpec
:
3764 # will use this after clone to set the variable
3765 self
.useClientSpec_from_options
= True
3767 if gitConfigBool("git-p4.useclientspec"):
3768 self
.useClientSpec
= True
3769 if self
.useClientSpec
:
3770 self
.clientSpecDirs
= getClientSpec()
3772 # TODO: should always look at previous commits,
3773 # merge with previous imports, if possible.
3776 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3778 # branches holds mapping from branch name to sha1
3779 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3781 # restrict to just this one, disabling detect-branches
3782 if branch_arg_given
:
3783 short
= shortP4Ref(self
.branch
, self
.importIntoRemotes
)
3784 if short
in branches
:
3785 self
.p4BranchesInGit
= [ short
]
3786 elif self
.branch
.startswith('refs/') and \
3787 branchExists(self
.branch
) and \
3788 '[git-p4:' in extractLogMessageFromGitCommit(self
.branch
):
3789 self
.p4BranchesInGit
= [ self
.branch
]
3791 self
.p4BranchesInGit
= branches
.keys()
3793 if len(self
.p4BranchesInGit
) > 1:
3795 print("Importing from/into multiple branches")
3796 self
.detectBranches
= True
3797 for branch
in branches
.keys():
3798 self
.initialParents
[self
.refPrefix
+ branch
] = \
3802 print("branches: %s" % self
.p4BranchesInGit
)
3805 for branch
in self
.p4BranchesInGit
:
3806 logMsg
= extractLogMessageFromGitCommit(fullP4Ref(branch
,
3807 self
.importIntoRemotes
))
3809 settings
= extractSettingsGitLog(logMsg
)
3811 self
.readOptions(settings
)
3812 if ('depot-paths' in settings
3813 and 'change' in settings
):
3814 change
= int(settings
['change']) + 1
3815 p4Change
= max(p4Change
, change
)
3817 depotPaths
= sorted(settings
['depot-paths'])
3818 if self
.previousDepotPaths
== []:
3819 self
.previousDepotPaths
= depotPaths
3822 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3823 prev_list
= prev
.split("/")
3824 cur_list
= cur
.split("/")
3825 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3826 if cur_list
[i
] != prev_list
[i
]:
3830 paths
.append ("/".join(cur_list
[:i
+ 1]))
3832 self
.previousDepotPaths
= paths
3835 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3836 self
.changeRange
= "@%s,#head" % p4Change
3837 if not self
.silent
and not self
.detectBranches
:
3838 print("Performing incremental import into %s git branch" % self
.branch
)
3840 self
.branch
= fullP4Ref(self
.branch
, self
.importIntoRemotes
)
3842 if len(args
) == 0 and self
.depotPaths
:
3844 print("Depot paths: %s" % ' '.join(self
.depotPaths
))
3846 if self
.depotPaths
and self
.depotPaths
!= args
:
3847 print("previous import used depot path %s and now %s was specified. "
3848 "This doesn't work!" % (' '.join (self
.depotPaths
),
3852 self
.depotPaths
= sorted(args
)
3857 # Make sure no revision specifiers are used when --changesfile
3859 bad_changesfile
= False
3860 if len(self
.changesFile
) > 0:
3861 for p
in self
.depotPaths
:
3862 if p
.find("@") >= 0 or p
.find("#") >= 0:
3863 bad_changesfile
= True
3866 die("Option --changesfile is incompatible with revision specifiers")
3869 for p
in self
.depotPaths
:
3870 if p
.find("@") != -1:
3871 atIdx
= p
.index("@")
3872 self
.changeRange
= p
[atIdx
:]
3873 if self
.changeRange
== "@all":
3874 self
.changeRange
= ""
3875 elif ',' not in self
.changeRange
:
3876 revision
= self
.changeRange
3877 self
.changeRange
= ""
3879 elif p
.find("#") != -1:
3880 hashIdx
= p
.index("#")
3881 revision
= p
[hashIdx
:]
3883 elif self
.previousDepotPaths
== []:
3884 # pay attention to changesfile, if given, else import
3885 # the entire p4 tree at the head revision
3886 if len(self
.changesFile
) == 0:
3889 p
= re
.sub ("\.\.\.$", "", p
)
3890 if not p
.endswith("/"):
3895 self
.depotPaths
= newPaths
3897 # --detect-branches may change this for each branch
3898 self
.branchPrefixes
= self
.depotPaths
3900 self
.loadUserMapFromCache()
3902 if self
.detectLabels
:
3905 if self
.detectBranches
:
3906 ## FIXME - what's a P4 projectName ?
3907 self
.projectName
= self
.guessProjectName()
3910 self
.getBranchMappingFromGitBranches()
3912 self
.getBranchMapping()
3914 print("p4-git branches: %s" % self
.p4BranchesInGit
)
3915 print("initial parents: %s" % self
.initialParents
)
3916 for b
in self
.p4BranchesInGit
:
3920 b
= b
[len(self
.projectName
):]
3921 self
.createdBranches
.add(b
)
3931 self
.importHeadRevision(revision
)
3933 self
.importRevisions(args
, branch_arg_given
)
3935 if gitConfigBool("git-p4.importLabels"):
3936 self
.importLabels
= True
3938 if self
.importLabels
:
3939 p4Labels
= getP4Labels(self
.depotPaths
)
3940 gitTags
= getGitTags()
3942 missingP4Labels
= p4Labels
- gitTags
3943 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3945 except P4CommandException
as e
:
3954 # Cleanup temporary branches created during import
3955 if self
.tempBranches
!= []:
3956 for branch
in self
.tempBranches
:
3957 read_pipe(["git", "update-ref", "-d", branch
])
3958 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3960 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3961 # a convenient shortcut refname "p4".
3962 if self
.importIntoRemotes
:
3963 head_ref
= self
.refPrefix
+ "HEAD"
3964 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3965 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3969 class P4Rebase(Command
):
3971 Command
.__init
__(self
)
3973 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3975 self
.importLabels
= False
3976 self
.description
= ("Fetches the latest revision from perforce and "
3977 + "rebases the current work (branch) against it")
3979 def run(self
, args
):
3981 sync
.importLabels
= self
.importLabels
3984 return self
.rebase()
3987 if os
.system("git update-index --refresh") != 0:
3988 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3989 if len(read_pipe(["git", "diff-index", "HEAD", "--"])) > 0:
3990 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3992 [upstream
, settings
] = findUpstreamBranchPoint()
3993 if len(upstream
) == 0:
3994 die("Cannot find upstream branchpoint for rebase")
3996 # the branchpoint may be p4/foo~3, so strip off the parent
3997 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3999 print("Rebasing the current branch onto %s" % upstream
)
4000 oldHead
= read_pipe(["git", "rev-parse", "HEAD"]).strip()
4001 system(["git", "rebase", upstream
])
4002 system(["git", "diff-tree", "--stat", "--summary", "-M", oldHead
,
4006 class P4Clone(P4Sync
):
4008 P4Sync
.__init
__(self
)
4009 self
.description
= "Creates a new git repository and imports from Perforce into it"
4010 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
4012 optparse
.make_option("--destination", dest
="cloneDestination",
4013 action
='store', default
=None,
4014 help="where to leave result of the clone"),
4015 optparse
.make_option("--bare", dest
="cloneBare",
4016 action
="store_true", default
=False),
4018 self
.cloneDestination
= None
4019 self
.needsGit
= False
4020 self
.cloneBare
= False
4022 def defaultDestination(self
, args
):
4023 ## TODO: use common prefix of args?
4025 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
4026 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
4027 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
4028 depotDir
= re
.sub(r
"/$", "", depotDir
)
4029 return os
.path
.split(depotDir
)[1]
4031 def run(self
, args
):
4035 if self
.keepRepoPath
and not self
.cloneDestination
:
4036 sys
.stderr
.write("Must specify destination for --keep-path\n")
4041 if not self
.cloneDestination
and len(depotPaths
) > 1:
4042 self
.cloneDestination
= depotPaths
[-1]
4043 depotPaths
= depotPaths
[:-1]
4045 for p
in depotPaths
:
4046 if not p
.startswith("//"):
4047 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
4050 if not self
.cloneDestination
:
4051 self
.cloneDestination
= self
.defaultDestination(args
)
4053 print("Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
))
4055 if not os
.path
.exists(self
.cloneDestination
):
4056 os
.makedirs(self
.cloneDestination
)
4057 chdir(self
.cloneDestination
)
4059 init_cmd
= [ "git", "init" ]
4061 init_cmd
.append("--bare")
4062 retcode
= subprocess
.call(init_cmd
)
4064 raise subprocess
.CalledProcessError(retcode
, init_cmd
)
4066 if not P4Sync
.run(self
, depotPaths
):
4069 # create a master branch and check out a work tree
4070 if gitBranchExists(self
.branch
):
4071 system([ "git", "branch", currentGitBranch(), self
.branch
])
4072 if not self
.cloneBare
:
4073 system([ "git", "checkout", "-f" ])
4075 print('Not checking out any branch, use ' \
4076 '"git checkout -q -b master <branch>"')
4078 # auto-set this variable if invoked with --use-client-spec
4079 if self
.useClientSpec_from_options
:
4080 system(["git", "config", "--bool", "git-p4.useclientspec", "true"])
4084 class P4Unshelve(Command
):
4086 Command
.__init
__(self
)
4088 self
.origin
= "HEAD"
4089 self
.description
= "Unshelve a P4 changelist into a git commit"
4090 self
.usage
= "usage: %prog [options] changelist"
4092 optparse
.make_option("--origin", dest
="origin",
4093 help="Use this base revision instead of the default (%s)" % self
.origin
),
4095 self
.verbose
= False
4096 self
.noCommit
= False
4097 self
.destbranch
= "refs/remotes/p4-unshelved"
4099 def renameBranch(self
, branch_name
):
4100 """ Rename the existing branch to branch_name.N
4104 for i
in range(0,1000):
4105 backup_branch_name
= "{0}.{1}".format(branch_name
, i
)
4106 if not gitBranchExists(backup_branch_name
):
4107 gitUpdateRef(backup_branch_name
, branch_name
) # copy ref to backup
4108 gitDeleteRef(branch_name
)
4110 print("renamed old unshelve branch to {0}".format(backup_branch_name
))
4114 sys
.exit("gave up trying to rename existing branch {0}".format(sync
.branch
))
4116 def findLastP4Revision(self
, starting_point
):
4117 """ Look back from starting_point for the first commit created by git-p4
4118 to find the P4 commit we are based on, and the depot-paths.
4121 for parent
in (range(65535)):
4122 log
= extractLogMessageFromGitCommit("{0}~{1}".format(starting_point
, parent
))
4123 settings
= extractSettingsGitLog(log
)
4124 if 'change' in settings
:
4127 sys
.exit("could not find git-p4 commits in {0}".format(self
.origin
))
4129 def createShelveParent(self
, change
, branch_name
, sync
, origin
):
4130 """ Create a commit matching the parent of the shelved changelist 'change'
4132 parent_description
= p4_describe(change
, shelved
=True)
4133 parent_description
['desc'] = 'parent for shelved changelist {}\n'.format(change
)
4134 files
= sync
.extractFilesFromCommit(parent_description
, shelved
=False, shelved_cl
=change
)
4138 # if it was added in the shelved changelist, it won't exist in the parent
4139 if f
['action'] in self
.add_actions
:
4142 # if it was deleted in the shelved changelist it must not be deleted
4143 # in the parent - we might even need to create it if the origin branch
4145 if f
['action'] in self
.delete_actions
:
4148 parent_files
.append(f
)
4150 sync
.commit(parent_description
, parent_files
, branch_name
,
4151 parent
=origin
, allow_empty
=True)
4152 print("created parent commit for {0} based on {1} in {2}".format(
4153 change
, self
.origin
, branch_name
))
4155 def run(self
, args
):
4159 if not gitBranchExists(self
.origin
):
4160 sys
.exit("origin branch {0} does not exist".format(self
.origin
))
4165 # only one change at a time
4168 # if the target branch already exists, rename it
4169 branch_name
= "{0}/{1}".format(self
.destbranch
, change
)
4170 if gitBranchExists(branch_name
):
4171 self
.renameBranch(branch_name
)
4172 sync
.branch
= branch_name
4174 sync
.verbose
= self
.verbose
4175 sync
.suppress_meta_comment
= True
4177 settings
= self
.findLastP4Revision(self
.origin
)
4178 sync
.depotPaths
= settings
['depot-paths']
4179 sync
.branchPrefixes
= sync
.depotPaths
4182 sync
.loadUserMapFromCache()
4185 # create a commit for the parent of the shelved changelist
4186 self
.createShelveParent(change
, branch_name
, sync
, self
.origin
)
4188 # create the commit for the shelved changelist itself
4189 description
= p4_describe(change
, True)
4190 files
= sync
.extractFilesFromCommit(description
, True, change
)
4192 sync
.commit(description
, files
, branch_name
, "")
4195 print("unshelved changelist {0} into {1}".format(change
, branch_name
))
4199 class P4Branches(Command
):
4201 Command
.__init
__(self
)
4203 self
.description
= ("Shows the git branches that hold imports and their "
4204 + "corresponding perforce depot paths")
4205 self
.verbose
= False
4207 def run(self
, args
):
4208 if originP4BranchesExist():
4209 createOrUpdateBranchesFromOrigin()
4211 for line
in read_pipe_lines(["git", "rev-parse", "--symbolic", "--remotes"]):
4214 if not line
.startswith('p4/') or line
== "p4/HEAD":
4218 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
4219 settings
= extractSettingsGitLog(log
)
4221 print("%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"]))
4224 class HelpFormatter(optparse
.IndentedHelpFormatter
):
4226 optparse
.IndentedHelpFormatter
.__init
__(self
)
4228 def format_description(self
, description
):
4230 return description
+ "\n"
4234 def printUsage(commands
):
4235 print("usage: %s <command> [options]" % sys
.argv
[0])
4237 print("valid commands: %s" % ", ".join(commands
))
4239 print("Try %s <command> --help for command specific help." % sys
.argv
[0])
4243 "submit" : P4Submit
,
4244 "commit" : P4Submit
,
4246 "rebase" : P4Rebase
,
4248 "branches" : P4Branches
,
4249 "unshelve" : P4Unshelve
,
4253 if len(sys
.argv
[1:]) == 0:
4254 printUsage(commands
.keys())
4257 cmdName
= sys
.argv
[1]
4259 klass
= commands
[cmdName
]
4262 print("unknown command %s" % cmdName
)
4264 printUsage(commands
.keys())
4267 options
= cmd
.options
4268 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
4272 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
4274 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
4276 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
4278 description
= cmd
.description
,
4279 formatter
= HelpFormatter())
4282 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
4288 verbose
= cmd
.verbose
4290 if cmd
.gitdir
== None:
4291 cmd
.gitdir
= os
.path
.abspath(".git")
4292 if not isValidGitDir(cmd
.gitdir
):
4293 # "rev-parse --git-dir" without arguments will try $PWD/.git
4294 cmd
.gitdir
= read_pipe(["git", "rev-parse", "--git-dir"]).strip()
4295 if os
.path
.exists(cmd
.gitdir
):
4296 cdup
= read_pipe(["git", "rev-parse", "--show-cdup"]).strip()
4300 if not isValidGitDir(cmd
.gitdir
):
4301 if isValidGitDir(cmd
.gitdir
+ "/.git"):
4302 cmd
.gitdir
+= "/.git"
4304 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
4306 # so git commands invoked from the P4 workspace will succeed
4307 os
.environ
["GIT_DIR"] = cmd
.gitdir
4309 if not cmd
.run(args
):
4314 if __name__
== '__main__':