3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
31 from subprocess
import CalledProcessError
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self
, returncode
, cmd
):
40 self
.returncode
= returncode
43 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
50 # The block size is reduced automatically if required
51 defaultBlockSize
= 1<<20
53 p4_access_checked
= False
55 def p4_build_cmd(cmd
):
56 """Build a suitable p4 command line.
58 This consolidates building and returning a p4 command line into one
59 location. It means that hooking into the environment, or other configuration
60 can be done more easily.
64 user
= gitConfig("git-p4.user")
66 real_cmd
+= ["-u",user
]
68 password
= gitConfig("git-p4.password")
70 real_cmd
+= ["-P", password
]
72 port
= gitConfig("git-p4.port")
74 real_cmd
+= ["-p", port
]
76 host
= gitConfig("git-p4.host")
78 real_cmd
+= ["-H", host
]
80 client
= gitConfig("git-p4.client")
82 real_cmd
+= ["-c", client
]
84 retries
= gitConfigInt("git-p4.retries")
86 # Perform 3 retries by default
89 # Provide a way to not pass this option by setting git-p4.retries to 0
90 real_cmd
+= ["-r", str(retries
)]
92 if isinstance(cmd
,basestring
):
93 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
97 # now check that we can actually talk to the server
98 global p4_access_checked
99 if not p4_access_checked
:
100 p4_access_checked
= True # suppress access checks in p4_check_access itself
106 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
107 This won't automatically add ".git" to a directory.
109 d
= read_pipe(["git", "--git-dir", path
, "rev-parse", "--git-dir"], True).strip()
110 if not d
or len(d
) == 0:
115 def chdir(path
, is_client_path
=False):
116 """Do chdir to the given path, and set the PWD environment
117 variable for use by P4. It does not look at getcwd() output.
118 Since we're not using the shell, it is necessary to set the
119 PWD environment variable explicitly.
121 Normally, expand the path to force it to be absolute. This
122 addresses the use of relative path names inside P4 settings,
123 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
124 as given; it looks for .p4config using PWD.
126 If is_client_path, the path was handed to us directly by p4,
127 and may be a symbolic link. Do not call os.getcwd() in this
128 case, because it will cause p4 to think that PWD is not inside
133 if not is_client_path
:
135 os
.environ
['PWD'] = path
138 """Return free space in bytes on the disk of the given dirname."""
139 if platform
.system() == 'Windows':
140 free_bytes
= ctypes
.c_ulonglong(0)
141 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
142 return free_bytes
.value
144 st
= os
.statvfs(os
.getcwd())
145 return st
.f_bavail
* st
.f_frsize
151 sys
.stderr
.write(msg
+ "\n")
154 def write_pipe(c
, stdin
):
156 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
158 expand
= isinstance(c
,basestring
)
159 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
161 val
= pipe
.write(stdin
)
164 die('Command failed: %s' % str(c
))
168 def p4_write_pipe(c
, stdin
):
169 real_cmd
= p4_build_cmd(c
)
170 return write_pipe(real_cmd
, stdin
)
172 def read_pipe_full(c
):
173 """ Read output from command. Returns a tuple
174 of the return status, stdout text and stderr
178 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
180 expand
= isinstance(c
,basestring
)
181 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
182 (out
, err
) = p
.communicate()
183 return (p
.returncode
, out
, err
)
185 def read_pipe(c
, ignore_error
=False):
186 """ Read output from command. Returns the output text on
187 success. On failure, terminates execution, unless
188 ignore_error is True, when it returns an empty string.
190 (retcode
, out
, err
) = read_pipe_full(c
)
195 die('Command failed: %s\nError: %s' % (str(c
), err
))
198 def read_pipe_text(c
):
199 """ Read output from a command with trailing whitespace stripped.
200 On error, returns None.
202 (retcode
, out
, err
) = read_pipe_full(c
)
208 def p4_read_pipe(c
, ignore_error
=False):
209 real_cmd
= p4_build_cmd(c
)
210 return read_pipe(real_cmd
, ignore_error
)
212 def read_pipe_lines(c
):
214 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
216 expand
= isinstance(c
, basestring
)
217 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
219 val
= pipe
.readlines()
220 if pipe
.close() or p
.wait():
221 die('Command failed: %s' % str(c
))
225 def p4_read_pipe_lines(c
):
226 """Specifically invoke p4 on the command supplied. """
227 real_cmd
= p4_build_cmd(c
)
228 return read_pipe_lines(real_cmd
)
230 def p4_has_command(cmd
):
231 """Ask p4 for help on this command. If it returns an error, the
232 command does not exist in this version of p4."""
233 real_cmd
= p4_build_cmd(["help", cmd
])
234 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
235 stderr
=subprocess
.PIPE
)
237 return p
.returncode
== 0
239 def p4_has_move_command():
240 """See if the move command exists, that it supports -k, and that
241 it has not been administratively disabled. The arguments
242 must be correct, but the filenames do not have to exist. Use
243 ones with wildcards so even if they exist, it will fail."""
245 if not p4_has_command("move"):
247 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
248 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
249 (out
, err
) = p
.communicate()
250 # return code will be 1 in either case
251 if err
.find("Invalid option") >= 0:
253 if err
.find("disabled") >= 0:
255 # assume it failed because @... was invalid changelist
258 def system(cmd
, ignore_error
=False):
259 expand
= isinstance(cmd
,basestring
)
261 sys
.stderr
.write("executing %s\n" % str(cmd
))
262 retcode
= subprocess
.call(cmd
, shell
=expand
)
263 if retcode
and not ignore_error
:
264 raise CalledProcessError(retcode
, cmd
)
269 """Specifically invoke p4 as the system command. """
270 real_cmd
= p4_build_cmd(cmd
)
271 expand
= isinstance(real_cmd
, basestring
)
272 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
274 raise CalledProcessError(retcode
, real_cmd
)
276 def die_bad_access(s
):
277 die("failure accessing depot: {0}".format(s
.rstrip()))
279 def p4_check_access(min_expiration
=1):
280 """ Check if we can access Perforce - account still logged in
282 results
= p4CmdList(["login", "-s"])
284 if len(results
) == 0:
285 # should never get here: always get either some results, or a p4ExitCode
286 assert("could not parse response from perforce")
290 if 'p4ExitCode' in result
:
291 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
292 die_bad_access("could not run p4")
294 code
= result
.get("code")
296 # we get here if we couldn't connect and there was nothing to unmarshal
297 die_bad_access("could not connect")
300 expiry
= result
.get("TicketExpiration")
303 if expiry
> min_expiration
:
307 die_bad_access("perforce ticket expires in {0} seconds".format(expiry
))
310 # account without a timeout - all ok
313 elif code
== "error":
314 data
= result
.get("data")
316 die_bad_access("p4 error: {0}".format(data
))
318 die_bad_access("unknown error")
320 die_bad_access("unknown error code {0}".format(code
))
322 _p4_version_string
= None
323 def p4_version_string():
324 """Read the version string, showing just the last line, which
325 hopefully is the interesting version bit.
328 Perforce - The Fast Software Configuration Management System.
329 Copyright 1995-2011 Perforce Software. All rights reserved.
330 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
332 global _p4_version_string
333 if not _p4_version_string
:
334 a
= p4_read_pipe_lines(["-V"])
335 _p4_version_string
= a
[-1].rstrip()
336 return _p4_version_string
338 def p4_integrate(src
, dest
):
339 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
341 def p4_sync(f
, *options
):
342 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
345 # forcibly add file names with wildcards
346 if wildcard_present(f
):
347 p4_system(["add", "-f", f
])
349 p4_system(["add", f
])
352 p4_system(["delete", wildcard_encode(f
)])
354 def p4_edit(f
, *options
):
355 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
358 p4_system(["revert", wildcard_encode(f
)])
360 def p4_reopen(type, f
):
361 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
363 def p4_reopen_in_change(changelist
, files
):
364 cmd
= ["reopen", "-c", str(changelist
)] + files
367 def p4_move(src
, dest
):
368 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
370 def p4_last_change():
371 results
= p4CmdList(["changes", "-m", "1"], skip_info
=True)
372 return int(results
[0]['change'])
374 def p4_describe(change
, shelved
=False):
375 """Make sure it returns a valid result by checking for
376 the presence of field "time". Return a dict of the
379 cmd
= ["describe", "-s"]
384 ds
= p4CmdList(cmd
, skip_info
=True)
386 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
390 if "p4ExitCode" in d
:
391 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
394 if d
["code"] == "error":
395 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
398 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
403 # Canonicalize the p4 type and return a tuple of the
404 # base type, plus any modifiers. See "p4 help filetypes"
405 # for a list and explanation.
407 def split_p4_type(p4type
):
409 p4_filetypes_historical
= {
410 "ctempobj": "binary+Sw",
416 "tempobj": "binary+FSw",
417 "ubinary": "binary+F",
418 "uresource": "resource+F",
419 "uxbinary": "binary+Fx",
420 "xbinary": "binary+x",
422 "xtempobj": "binary+Swx",
424 "xunicode": "unicode+x",
427 if p4type
in p4_filetypes_historical
:
428 p4type
= p4_filetypes_historical
[p4type
]
430 s
= p4type
.split("+")
438 # return the raw p4 type of a file (text, text+ko, etc)
441 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
442 return results
[0]['headType']
445 # Given a type base and modifier, return a regexp matching
446 # the keywords that can be expanded in the file
448 def p4_keywords_regexp_for_type(base
, type_mods
):
449 if base
in ("text", "unicode", "binary"):
451 if "ko" in type_mods
:
453 elif "k" in type_mods
:
454 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
458 \$ # Starts with a dollar, followed by...
459 (%s) # one of the keywords, followed by...
460 (:[^$\n]+)? # possibly an old expansion, followed by...
468 # Given a file, return a regexp matching the possible
469 # RCS keywords that will be expanded, or None for files
470 # with kw expansion turned off.
472 def p4_keywords_regexp_for_file(file):
473 if not os
.path
.exists(file):
476 (type_base
, type_mods
) = split_p4_type(p4_type(file))
477 return p4_keywords_regexp_for_type(type_base
, type_mods
)
479 def setP4ExecBit(file, mode
):
480 # Reopens an already open file and changes the execute bit to match
481 # the execute bit setting in the passed in mode.
485 if not isModeExec(mode
):
486 p4Type
= getP4OpenedType(file)
487 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
488 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
489 if p4Type
[-1] == "+":
490 p4Type
= p4Type
[0:-1]
492 p4_reopen(p4Type
, file)
494 def getP4OpenedType(file):
495 # Returns the perforce file type for the given file.
497 result
= p4_read_pipe(["opened", wildcard_encode(file)])
498 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
500 return match
.group(1)
502 die("Could not determine file type for %s (result: '%s')" % (file, result
))
504 # Return the set of all p4 labels
505 def getP4Labels(depotPaths
):
507 if isinstance(depotPaths
,basestring
):
508 depotPaths
= [depotPaths
]
510 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
516 # Return the set of all git tags
519 for line
in read_pipe_lines(["git", "tag"]):
524 def diffTreePattern():
525 # This is a simple generator for the diff tree regex pattern. This could be
526 # a class variable if this and parseDiffTreeEntry were a part of a class.
527 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
531 def parseDiffTreeEntry(entry
):
532 """Parses a single diff tree entry into its component elements.
534 See git-diff-tree(1) manpage for details about the format of the diff
535 output. This method returns a dictionary with the following elements:
537 src_mode - The mode of the source file
538 dst_mode - The mode of the destination file
539 src_sha1 - The sha1 for the source file
540 dst_sha1 - The sha1 fr the destination file
541 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
542 status_score - The score for the status (applicable for 'C' and 'R'
543 statuses). This is None if there is no score.
544 src - The path for the source file.
545 dst - The path for the destination file. This is only present for
546 copy or renames. If it is not present, this is None.
548 If the pattern is not matched, None is returned."""
550 match
= diffTreePattern().next().match(entry
)
553 'src_mode': match
.group(1),
554 'dst_mode': match
.group(2),
555 'src_sha1': match
.group(3),
556 'dst_sha1': match
.group(4),
557 'status': match
.group(5),
558 'status_score': match
.group(6),
559 'src': match
.group(7),
560 'dst': match
.group(10)
564 def isModeExec(mode
):
565 # Returns True if the given git mode represents an executable file,
567 return mode
[-3:] == "755"
569 class P4Exception(Exception):
570 """ Base class for exceptions from the p4 client """
571 def __init__(self
, exit_code
):
572 self
.p4ExitCode
= exit_code
574 class P4ServerException(P4Exception
):
575 """ Base class for exceptions where we get some kind of marshalled up result from the server """
576 def __init__(self
, exit_code
, p4_result
):
577 super(P4ServerException
, self
).__init
__(exit_code
)
578 self
.p4_result
= p4_result
579 self
.code
= p4_result
[0]['code']
580 self
.data
= p4_result
[0]['data']
582 class P4RequestSizeException(P4ServerException
):
583 """ One of the maxresults or maxscanrows errors """
584 def __init__(self
, exit_code
, p4_result
, limit
):
585 super(P4RequestSizeException
, self
).__init
__(exit_code
, p4_result
)
588 def isModeExecChanged(src_mode
, dst_mode
):
589 return isModeExec(src_mode
) != isModeExec(dst_mode
)
591 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None, skip_info
=False,
592 errors_as_exceptions
=False):
594 if isinstance(cmd
,basestring
):
601 cmd
= p4_build_cmd(cmd
)
603 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
605 # Use a temporary file to avoid deadlocks without
606 # subprocess.communicate(), which would put another copy
607 # of stdout into memory.
609 if stdin
is not None:
610 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
611 if isinstance(stdin
,basestring
):
612 stdin_file
.write(stdin
)
615 stdin_file
.write(i
+ '\n')
619 p4
= subprocess
.Popen(cmd
,
622 stdout
=subprocess
.PIPE
)
627 entry
= marshal
.load(p4
.stdout
)
629 if 'code' in entry
and entry
['code'] == 'info':
639 if errors_as_exceptions
:
641 data
= result
[0].get('data')
643 m
= re
.search('Too many rows scanned \(over (\d+)\)', data
)
645 m
= re
.search('Request too large \(over (\d+)\)', data
)
648 limit
= int(m
.group(1))
649 raise P4RequestSizeException(exitCode
, result
, limit
)
651 raise P4ServerException(exitCode
, result
)
653 raise P4Exception(exitCode
)
656 entry
["p4ExitCode"] = exitCode
662 list = p4CmdList(cmd
)
668 def p4Where(depotPath
):
669 if not depotPath
.endswith("/"):
671 depotPathLong
= depotPath
+ "..."
672 outputList
= p4CmdList(["where", depotPathLong
])
674 for entry
in outputList
:
675 if "depotFile" in entry
:
676 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
677 # The base path always ends with "/...".
678 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
681 elif "data" in entry
:
682 data
= entry
.get("data")
683 space
= data
.find(" ")
684 if data
[:space
] == depotPath
:
689 if output
["code"] == "error":
693 clientPath
= output
.get("path")
694 elif "data" in output
:
695 data
= output
.get("data")
696 lastSpace
= data
.rfind(" ")
697 clientPath
= data
[lastSpace
+ 1:]
699 if clientPath
.endswith("..."):
700 clientPath
= clientPath
[:-3]
703 def currentGitBranch():
704 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
706 def isValidGitDir(path
):
707 return git_dir(path
) != None
709 def parseRevision(ref
):
710 return read_pipe("git rev-parse %s" % ref
).strip()
712 def branchExists(ref
):
713 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
717 def extractLogMessageFromGitCommit(commit
):
720 ## fixme: title is first line of commit, not 1st paragraph.
722 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
731 def extractSettingsGitLog(log
):
733 for line
in log
.split("\n"):
735 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
739 assignments
= m
.group(1).split (':')
740 for a
in assignments
:
742 key
= vals
[0].strip()
743 val
= ('='.join (vals
[1:])).strip()
744 if val
.endswith ('\"') and val
.startswith('"'):
749 paths
= values
.get("depot-paths")
751 paths
= values
.get("depot-path")
753 values
['depot-paths'] = paths
.split(',')
756 def gitBranchExists(branch
):
757 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
758 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
759 return proc
.wait() == 0;
761 def gitUpdateRef(ref
, newvalue
):
762 subprocess
.check_call(["git", "update-ref", ref
, newvalue
])
764 def gitDeleteRef(ref
):
765 subprocess
.check_call(["git", "update-ref", "-d", ref
])
769 def gitConfig(key
, typeSpecifier
=None):
770 if key
not in _gitConfig
:
771 cmd
= [ "git", "config" ]
773 cmd
+= [ typeSpecifier
]
775 s
= read_pipe(cmd
, ignore_error
=True)
776 _gitConfig
[key
] = s
.strip()
777 return _gitConfig
[key
]
779 def gitConfigBool(key
):
780 """Return a bool, using git config --bool. It is True only if the
781 variable is set to true, and False if set to false or not present
784 if key
not in _gitConfig
:
785 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
786 return _gitConfig
[key
]
788 def gitConfigInt(key
):
789 if key
not in _gitConfig
:
790 cmd
= [ "git", "config", "--int", key
]
791 s
= read_pipe(cmd
, ignore_error
=True)
794 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
796 _gitConfig
[key
] = None
797 return _gitConfig
[key
]
799 def gitConfigList(key
):
800 if key
not in _gitConfig
:
801 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
802 _gitConfig
[key
] = s
.strip().splitlines()
803 if _gitConfig
[key
] == ['']:
805 return _gitConfig
[key
]
807 def p4BranchesInGit(branchesAreInRemotes
=True):
808 """Find all the branches whose names start with "p4/", looking
809 in remotes or heads as specified by the argument. Return
810 a dictionary of { branch: revision } for each one found.
811 The branch names are the short names, without any
816 cmdline
= "git rev-parse --symbolic "
817 if branchesAreInRemotes
:
818 cmdline
+= "--remotes"
820 cmdline
+= "--branches"
822 for line
in read_pipe_lines(cmdline
):
826 if not line
.startswith('p4/'):
828 # special symbolic ref to p4/master
829 if line
== "p4/HEAD":
832 # strip off p4/ prefix
833 branch
= line
[len("p4/"):]
835 branches
[branch
] = parseRevision(line
)
839 def branch_exists(branch
):
840 """Make sure that the given ref name really exists."""
842 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
843 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
844 out
, _
= p
.communicate()
847 # expect exactly one line of output: the branch name
848 return out
.rstrip() == branch
850 def findUpstreamBranchPoint(head
= "HEAD"):
851 branches
= p4BranchesInGit()
852 # map from depot-path to branch name
853 branchByDepotPath
= {}
854 for branch
in branches
.keys():
855 tip
= branches
[branch
]
856 log
= extractLogMessageFromGitCommit(tip
)
857 settings
= extractSettingsGitLog(log
)
858 if "depot-paths" in settings
:
859 paths
= ",".join(settings
["depot-paths"])
860 branchByDepotPath
[paths
] = "remotes/p4/" + branch
864 while parent
< 65535:
865 commit
= head
+ "~%s" % parent
866 log
= extractLogMessageFromGitCommit(commit
)
867 settings
= extractSettingsGitLog(log
)
868 if "depot-paths" in settings
:
869 paths
= ",".join(settings
["depot-paths"])
870 if paths
in branchByDepotPath
:
871 return [branchByDepotPath
[paths
], settings
]
875 return ["", settings
]
877 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
879 print ("Creating/updating branch(es) in %s based on origin branch(es)"
882 originPrefix
= "origin/p4/"
884 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
886 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
889 headName
= line
[len(originPrefix
):]
890 remoteHead
= localRefPrefix
+ headName
893 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
894 if ('depot-paths' not in original
895 or 'change' not in original
):
899 if not gitBranchExists(remoteHead
):
901 print "creating %s" % remoteHead
904 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
905 if 'change' in settings
:
906 if settings
['depot-paths'] == original
['depot-paths']:
907 originP4Change
= int(original
['change'])
908 p4Change
= int(settings
['change'])
909 if originP4Change
> p4Change
:
910 print ("%s (%s) is newer than %s (%s). "
911 "Updating p4 branch from origin."
912 % (originHead
, originP4Change
,
913 remoteHead
, p4Change
))
916 print ("Ignoring: %s was imported from %s while "
917 "%s was imported from %s"
918 % (originHead
, ','.join(original
['depot-paths']),
919 remoteHead
, ','.join(settings
['depot-paths'])))
922 system("git update-ref %s %s" % (remoteHead
, originHead
))
924 def originP4BranchesExist():
925 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
928 def p4ParseNumericChangeRange(parts
):
929 changeStart
= int(parts
[0][1:])
930 if parts
[1] == '#head':
931 changeEnd
= p4_last_change()
933 changeEnd
= int(parts
[1])
935 return (changeStart
, changeEnd
)
937 def chooseBlockSize(blockSize
):
941 return defaultBlockSize
943 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
946 # Parse the change range into start and end. Try to find integer
947 # revision ranges as these can be broken up into blocks to avoid
948 # hitting server-side limits (maxrows, maxscanresults). But if
949 # that doesn't work, fall back to using the raw revision specifier
950 # strings, without using block mode.
952 if changeRange
is None or changeRange
== '':
954 changeEnd
= p4_last_change()
955 block_size
= chooseBlockSize(requestedBlockSize
)
957 parts
= changeRange
.split(',')
958 assert len(parts
) == 2
960 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
961 block_size
= chooseBlockSize(requestedBlockSize
)
963 changeStart
= parts
[0][1:]
965 if requestedBlockSize
:
966 die("cannot use --changes-block-size with non-numeric revisions")
971 # Retrieve changes a block at a time, to prevent running
972 # into a MaxResults/MaxScanRows error from the server. If
973 # we _do_ hit one of those errors, turn down the block size
979 end
= min(changeEnd
, changeStart
+ block_size
)
980 revisionRange
= "%d,%d" % (changeStart
, end
)
982 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
985 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
989 result
= p4CmdList(cmd
, errors_as_exceptions
=True)
990 except P4RequestSizeException
as e
:
993 elif block_size
> e
.limit
:
996 block_size
= max(2, block_size
// 2)
998 if verbose
: print("block size error, retrying with block size {0}".format(block_size
))
1000 except P4Exception
as e
:
1001 die('Error retrieving changes description ({0})'.format(e
.p4ExitCode
))
1003 # Insert changes in chronological order
1004 for entry
in reversed(result
):
1005 if 'change' not in entry
:
1007 changes
.add(int(entry
['change']))
1012 if end
>= changeEnd
:
1015 changeStart
= end
+ 1
1017 changes
= sorted(changes
)
1020 def p4PathStartsWith(path
, prefix
):
1021 # This method tries to remedy a potential mixed-case issue:
1023 # If UserA adds //depot/DirA/file1
1024 # and UserB adds //depot/dira/file2
1026 # we may or may not have a problem. If you have core.ignorecase=true,
1027 # we treat DirA and dira as the same directory
1028 if gitConfigBool("core.ignorecase"):
1029 return path
.lower().startswith(prefix
.lower())
1030 return path
.startswith(prefix
)
1032 def getClientSpec():
1033 """Look at the p4 client spec, create a View() object that contains
1034 all the mappings, and return it."""
1036 specList
= p4CmdList("client -o")
1037 if len(specList
) != 1:
1038 die('Output from "client -o" is %d lines, expecting 1' %
1041 # dictionary of all client parameters
1044 # the //client/ name
1045 client_name
= entry
["Client"]
1047 # just the keys that start with "View"
1048 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
1050 # hold this new View
1051 view
= View(client_name
)
1053 # append the lines, in order, to the view
1054 for view_num
in range(len(view_keys
)):
1055 k
= "View%d" % view_num
1056 if k
not in view_keys
:
1057 die("Expected view key %s missing" % k
)
1058 view
.append(entry
[k
])
1062 def getClientRoot():
1063 """Grab the client directory."""
1065 output
= p4CmdList("client -o")
1066 if len(output
) != 1:
1067 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
1070 if "Root" not in entry
:
1071 die('Client has no "Root"')
1073 return entry
["Root"]
1076 # P4 wildcards are not allowed in filenames. P4 complains
1077 # if you simply add them, but you can force it with "-f", in
1078 # which case it translates them into %xx encoding internally.
1080 def wildcard_decode(path
):
1081 # Search for and fix just these four characters. Do % last so
1082 # that fixing it does not inadvertently create new %-escapes.
1083 # Cannot have * in a filename in windows; untested as to
1084 # what p4 would do in such a case.
1085 if not platform
.system() == "Windows":
1086 path
= path
.replace("%2A", "*")
1087 path
= path
.replace("%23", "#") \
1088 .replace("%40", "@") \
1089 .replace("%25", "%")
1092 def wildcard_encode(path
):
1093 # do % first to avoid double-encoding the %s introduced here
1094 path
= path
.replace("%", "%25") \
1095 .replace("*", "%2A") \
1096 .replace("#", "%23") \
1097 .replace("@", "%40")
1100 def wildcard_present(path
):
1101 m
= re
.search("[*#@%]", path
)
1102 return m
is not None
1104 class LargeFileSystem(object):
1105 """Base class for large file system support."""
1107 def __init__(self
, writeToGitStream
):
1108 self
.largeFiles
= set()
1109 self
.writeToGitStream
= writeToGitStream
1111 def generatePointer(self
, cloneDestination
, contentFile
):
1112 """Return the content of a pointer file that is stored in Git instead of
1113 the actual content."""
1114 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
1116 def pushFile(self
, localLargeFile
):
1117 """Push the actual content which is not stored in the Git repository to
1119 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
1121 def hasLargeFileExtension(self
, relPath
):
1123 lambda a
, b
: a
or b
,
1124 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
1128 def generateTempFile(self
, contents
):
1129 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1131 contentFile
.write(d
)
1133 return contentFile
.name
1135 def exceedsLargeFileThreshold(self
, relPath
, contents
):
1136 if gitConfigInt('git-p4.largeFileThreshold'):
1137 contentsSize
= sum(len(d
) for d
in contents
)
1138 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
1140 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1141 contentsSize
= sum(len(d
) for d
in contents
)
1142 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1144 contentTempFile
= self
.generateTempFile(contents
)
1145 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1146 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
1147 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
1149 compressedContentsSize
= zf
.infolist()[0].compress_size
1150 os
.remove(contentTempFile
)
1151 os
.remove(compressedContentFile
.name
)
1152 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
1156 def addLargeFile(self
, relPath
):
1157 self
.largeFiles
.add(relPath
)
1159 def removeLargeFile(self
, relPath
):
1160 self
.largeFiles
.remove(relPath
)
1162 def isLargeFile(self
, relPath
):
1163 return relPath
in self
.largeFiles
1165 def processContent(self
, git_mode
, relPath
, contents
):
1166 """Processes the content of git fast import. This method decides if a
1167 file is stored in the large file system and handles all necessary
1169 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1170 contentTempFile
= self
.generateTempFile(contents
)
1171 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1172 if pointer_git_mode
:
1173 git_mode
= pointer_git_mode
1175 # Move temp file to final location in large file system
1176 largeFileDir
= os
.path
.dirname(localLargeFile
)
1177 if not os
.path
.isdir(largeFileDir
):
1178 os
.makedirs(largeFileDir
)
1179 shutil
.move(contentTempFile
, localLargeFile
)
1180 self
.addLargeFile(relPath
)
1181 if gitConfigBool('git-p4.largeFilePush'):
1182 self
.pushFile(localLargeFile
)
1184 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1185 return (git_mode
, contents
)
1187 class MockLFS(LargeFileSystem
):
1188 """Mock large file system for testing."""
1190 def generatePointer(self
, contentFile
):
1191 """The pointer content is the original content prefixed with "pointer-".
1192 The local filename of the large file storage is derived from the file content.
1194 with
open(contentFile
, 'r') as f
:
1197 pointerContents
= 'pointer-' + content
1198 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1199 return (gitMode
, pointerContents
, localLargeFile
)
1201 def pushFile(self
, localLargeFile
):
1202 """The remote filename of the large file storage is the same as the local
1203 one but in a different directory.
1205 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1206 if not os
.path
.exists(remotePath
):
1207 os
.makedirs(remotePath
)
1208 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1210 class GitLFS(LargeFileSystem
):
1211 """Git LFS as backend for the git-p4 large file system.
1212 See https://git-lfs.github.com/ for details."""
1214 def __init__(self
, *args
):
1215 LargeFileSystem
.__init
__(self
, *args
)
1216 self
.baseGitAttributes
= []
1218 def generatePointer(self
, contentFile
):
1219 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1220 mode and content which is stored in the Git repository instead of
1221 the actual content. Return also the new location of the actual
1224 if os
.path
.getsize(contentFile
) == 0:
1225 return (None, '', None)
1227 pointerProcess
= subprocess
.Popen(
1228 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1229 stdout
=subprocess
.PIPE
1231 pointerFile
= pointerProcess
.stdout
.read()
1232 if pointerProcess
.wait():
1233 os
.remove(contentFile
)
1234 die('git-lfs pointer command failed. Did you install the extension?')
1236 # Git LFS removed the preamble in the output of the 'pointer' command
1237 # starting from version 1.2.0. Check for the preamble here to support
1239 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1240 if pointerFile
.startswith('Git LFS pointer for'):
1241 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1243 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1244 localLargeFile
= os
.path
.join(
1246 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1249 # LFS Spec states that pointer files should not have the executable bit set.
1251 return (gitMode
, pointerFile
, localLargeFile
)
1253 def pushFile(self
, localLargeFile
):
1254 uploadProcess
= subprocess
.Popen(
1255 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1257 if uploadProcess
.wait():
1258 die('git-lfs push command failed. Did you define a remote?')
1260 def generateGitAttributes(self
):
1262 self
.baseGitAttributes
+
1266 '# Git LFS (see https://git-lfs.github.com/)\n',
1269 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1270 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1272 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1273 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1277 def addLargeFile(self
, relPath
):
1278 LargeFileSystem
.addLargeFile(self
, relPath
)
1279 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1281 def removeLargeFile(self
, relPath
):
1282 LargeFileSystem
.removeLargeFile(self
, relPath
)
1283 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1285 def processContent(self
, git_mode
, relPath
, contents
):
1286 if relPath
== '.gitattributes':
1287 self
.baseGitAttributes
= contents
1288 return (git_mode
, self
.generateGitAttributes())
1290 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1294 self
.usage
= "usage: %prog [options]"
1295 self
.needsGit
= True
1296 self
.verbose
= False
1298 # This is required for the "append" cloneExclude action
1299 def ensure_value(self
, attr
, value
):
1300 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
1301 setattr(self
, attr
, value
)
1302 return getattr(self
, attr
)
1306 self
.userMapFromPerforceServer
= False
1307 self
.myP4UserId
= None
1311 return self
.myP4UserId
1313 results
= p4CmdList("user -o")
1316 self
.myP4UserId
= r
['User']
1318 die("Could not find your p4 user id")
1320 def p4UserIsMe(self
, p4User
):
1321 # return True if the given p4 user is actually me
1322 me
= self
.p4UserId()
1323 if not p4User
or p4User
!= me
:
1328 def getUserCacheFilename(self
):
1329 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1330 return home
+ "/.gitp4-usercache.txt"
1332 def getUserMapFromPerforceServer(self
):
1333 if self
.userMapFromPerforceServer
:
1338 for output
in p4CmdList("users"):
1339 if "User" not in output
:
1341 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1342 self
.emails
[output
["Email"]] = output
["User"]
1344 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1345 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1346 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1347 if mapUser
and len(mapUser
[0]) == 3:
1348 user
= mapUser
[0][0]
1349 fullname
= mapUser
[0][1]
1350 email
= mapUser
[0][2]
1351 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1352 self
.emails
[email
] = user
1355 for (key
, val
) in self
.users
.items():
1356 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1358 open(self
.getUserCacheFilename(), "wb").write(s
)
1359 self
.userMapFromPerforceServer
= True
1361 def loadUserMapFromCache(self
):
1363 self
.userMapFromPerforceServer
= False
1365 cache
= open(self
.getUserCacheFilename(), "rb")
1366 lines
= cache
.readlines()
1369 entry
= line
.strip().split("\t")
1370 self
.users
[entry
[0]] = entry
[1]
1372 self
.getUserMapFromPerforceServer()
1374 class P4Debug(Command
):
1376 Command
.__init
__(self
)
1378 self
.description
= "A tool to debug the output of p4 -G."
1379 self
.needsGit
= False
1381 def run(self
, args
):
1383 for output
in p4CmdList(args
):
1384 print 'Element: %d' % j
1389 class P4RollBack(Command
):
1391 Command
.__init
__(self
)
1393 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1395 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1396 self
.rollbackLocalBranches
= False
1398 def run(self
, args
):
1401 maxChange
= int(args
[0])
1403 if "p4ExitCode" in p4Cmd("changes -m 1"):
1404 die("Problems executing p4");
1406 if self
.rollbackLocalBranches
:
1407 refPrefix
= "refs/heads/"
1408 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1410 refPrefix
= "refs/remotes/"
1411 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1414 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1416 ref
= refPrefix
+ line
1417 log
= extractLogMessageFromGitCommit(ref
)
1418 settings
= extractSettingsGitLog(log
)
1420 depotPaths
= settings
['depot-paths']
1421 change
= settings
['change']
1425 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1426 for p
in depotPaths
]))) == 0:
1427 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1428 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1431 while change
and int(change
) > maxChange
:
1434 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1435 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1436 log
= extractLogMessageFromGitCommit(ref
)
1437 settings
= extractSettingsGitLog(log
)
1440 depotPaths
= settings
['depot-paths']
1441 change
= settings
['change']
1444 print "%s rewound to %s" % (ref
, change
)
1448 class P4Submit(Command
, P4UserMap
):
1450 conflict_behavior_choices
= ("ask", "skip", "quit")
1453 Command
.__init
__(self
)
1454 P4UserMap
.__init
__(self
)
1456 optparse
.make_option("--origin", dest
="origin"),
1457 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1458 # preserve the user, requires relevant p4 permissions
1459 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1460 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1461 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1462 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1463 optparse
.make_option("--conflict", dest
="conflict_behavior",
1464 choices
=self
.conflict_behavior_choices
),
1465 optparse
.make_option("--branch", dest
="branch"),
1466 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1467 help="Shelve instead of submit. Shelved files are reverted, "
1468 "restoring the workspace to the state before the shelve"),
1469 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="append", type="int",
1470 metavar
="CHANGELIST",
1471 help="update an existing shelved changelist, implies --shelve, "
1472 "repeat in-order for multiple shelved changelists"),
1473 optparse
.make_option("--commit", dest
="commit", metavar
="COMMIT",
1474 help="submit only the specified commit(s), one commit or xxx..xxx"),
1475 optparse
.make_option("--disable-rebase", dest
="disable_rebase", action
="store_true",
1476 help="Disable rebase after submit is completed. Can be useful if you "
1477 "work from a local git branch that is not master"),
1478 optparse
.make_option("--disable-p4sync", dest
="disable_p4sync", action
="store_true",
1479 help="Skip Perforce sync of p4/master after submit or shelve"),
1481 self
.description
= "Submit changes from git to the perforce depot."
1482 self
.usage
+= " [name of git branch to submit into perforce depot]"
1484 self
.detectRenames
= False
1485 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1486 self
.dry_run
= False
1488 self
.update_shelve
= list()
1490 self
.disable_rebase
= gitConfigBool("git-p4.disableRebase")
1491 self
.disable_p4sync
= gitConfigBool("git-p4.disableP4Sync")
1492 self
.prepare_p4_only
= False
1493 self
.conflict_behavior
= None
1494 self
.isWindows
= (platform
.system() == "Windows")
1495 self
.exportLabels
= False
1496 self
.p4HasMoveCommand
= p4_has_move_command()
1499 if gitConfig('git-p4.largeFileSystem'):
1500 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1503 if len(p4CmdList("opened ...")) > 0:
1504 die("You have files opened with perforce! Close them before starting the sync.")
1506 def separate_jobs_from_description(self
, message
):
1507 """Extract and return a possible Jobs field in the commit
1508 message. It goes into a separate section in the p4 change
1511 A jobs line starts with "Jobs:" and looks like a new field
1512 in a form. Values are white-space separated on the same
1513 line or on following lines that start with a tab.
1515 This does not parse and extract the full git commit message
1516 like a p4 form. It just sees the Jobs: line as a marker
1517 to pass everything from then on directly into the p4 form,
1518 but outside the description section.
1520 Return a tuple (stripped log message, jobs string)."""
1522 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1524 return (message
, None)
1526 jobtext
= message
[m
.start():]
1527 stripped_message
= message
[:m
.start()].rstrip()
1528 return (stripped_message
, jobtext
)
1530 def prepareLogMessage(self
, template
, message
, jobs
):
1531 """Edits the template returned from "p4 change -o" to insert
1532 the message in the Description field, and the jobs text in
1536 inDescriptionSection
= False
1538 for line
in template
.split("\n"):
1539 if line
.startswith("#"):
1540 result
+= line
+ "\n"
1543 if inDescriptionSection
:
1544 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1545 inDescriptionSection
= False
1546 # insert Jobs section
1548 result
+= jobs
+ "\n"
1552 if line
.startswith("Description:"):
1553 inDescriptionSection
= True
1555 for messageLine
in message
.split("\n"):
1556 line
+= "\t" + messageLine
+ "\n"
1558 result
+= line
+ "\n"
1562 def patchRCSKeywords(self
, file, pattern
):
1563 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1564 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1566 outFile
= os
.fdopen(handle
, "w+")
1567 inFile
= open(file, "r")
1568 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1569 for line
in inFile
.readlines():
1570 line
= regexp
.sub(r
'$\1$', line
)
1574 # Forcibly overwrite the original file
1576 shutil
.move(outFileName
, file)
1578 # cleanup our temporary file
1579 os
.unlink(outFileName
)
1580 print "Failed to strip RCS keywords in %s" % file
1583 print "Patched up RCS keywords in %s" % file
1585 def p4UserForCommit(self
,id):
1586 # Return the tuple (perforce user,git email) for a given git commit id
1587 self
.getUserMapFromPerforceServer()
1588 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1589 "--format=%ae", id])
1590 gitEmail
= gitEmail
.strip()
1591 if gitEmail
not in self
.emails
:
1592 return (None,gitEmail
)
1594 return (self
.emails
[gitEmail
],gitEmail
)
1596 def checkValidP4Users(self
,commits
):
1597 # check if any git authors cannot be mapped to p4 users
1599 (user
,email
) = self
.p4UserForCommit(id)
1601 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1602 if gitConfigBool("git-p4.allowMissingP4Users"):
1605 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1607 def lastP4Changelist(self
):
1608 # Get back the last changelist number submitted in this client spec. This
1609 # then gets used to patch up the username in the change. If the same
1610 # client spec is being used by multiple processes then this might go
1612 results
= p4CmdList("client -o") # find the current client
1616 client
= r
['Client']
1619 die("could not get client spec")
1620 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1624 die("Could not get changelist number for last submit - cannot patch up user details")
1626 def modifyChangelistUser(self
, changelist
, newUser
):
1627 # fixup the user field of a changelist after it has been submitted.
1628 changes
= p4CmdList("change -o %s" % changelist
)
1629 if len(changes
) != 1:
1630 die("Bad output from p4 change modifying %s to user %s" %
1631 (changelist
, newUser
))
1634 if c
['User'] == newUser
: return # nothing to do
1636 input = marshal
.dumps(c
)
1638 result
= p4CmdList("change -f -i", stdin
=input)
1641 if r
['code'] == 'error':
1642 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1644 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1646 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1648 def canChangeChangelists(self
):
1649 # check to see if we have p4 admin or super-user permissions, either of
1650 # which are required to modify changelists.
1651 results
= p4CmdList(["protects", self
.depotPath
])
1654 if r
['perm'] == 'admin':
1656 if r
['perm'] == 'super':
1660 def prepareSubmitTemplate(self
, changelist
=None):
1661 """Run "p4 change -o" to grab a change specification template.
1662 This does not use "p4 -G", as it is nice to keep the submission
1663 template in original order, since a human might edit it.
1665 Remove lines in the Files section that show changes to files
1666 outside the depot path we're committing into."""
1668 [upstream
, settings
] = findUpstreamBranchPoint()
1671 # A Perforce Change Specification.
1673 # Change: The change number. 'new' on a new changelist.
1674 # Date: The date this specification was last modified.
1675 # Client: The client on which the changelist was created. Read-only.
1676 # User: The user who created the changelist.
1677 # Status: Either 'pending' or 'submitted'. Read-only.
1678 # Type: Either 'public' or 'restricted'. Default is 'public'.
1679 # Description: Comments about the changelist. Required.
1680 # Jobs: What opened jobs are to be closed by this changelist.
1681 # You may delete jobs from this list. (New changelists only.)
1682 # Files: What opened files from the default changelist are to be added
1683 # to this changelist. You may delete files from this list.
1684 # (New changelists only.)
1687 inFilesSection
= False
1689 args
= ['change', '-o']
1691 args
.append(str(changelist
))
1692 for entry
in p4CmdList(args
):
1693 if 'code' not in entry
:
1695 if entry
['code'] == 'stat':
1696 change_entry
= entry
1698 if not change_entry
:
1699 die('Failed to decode output of p4 change -o')
1700 for key
, value
in change_entry
.iteritems():
1701 if key
.startswith('File'):
1702 if 'depot-paths' in settings
:
1703 if not [p
for p
in settings
['depot-paths']
1704 if p4PathStartsWith(value
, p
)]:
1707 if not p4PathStartsWith(value
, self
.depotPath
):
1709 files_list
.append(value
)
1711 # Output in the order expected by prepareLogMessage
1712 for key
in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1713 if key
not in change_entry
:
1716 template
+= key
+ ':'
1717 if key
== 'Description':
1719 for field_line
in change_entry
[key
].splitlines():
1720 template
+= '\t'+field_line
+'\n'
1721 if len(files_list
) > 0:
1723 template
+= 'Files:\n'
1724 for path
in files_list
:
1725 template
+= '\t'+path
+'\n'
1728 def edit_template(self
, template_file
):
1729 """Invoke the editor to let the user change the submission
1730 message. Return true if okay to continue with the submit."""
1732 # if configured to skip the editing part, just submit
1733 if gitConfigBool("git-p4.skipSubmitEdit"):
1736 # look at the modification time, to check later if the user saved
1738 mtime
= os
.stat(template_file
).st_mtime
1741 if "P4EDITOR" in os
.environ
and (os
.environ
.get("P4EDITOR") != ""):
1742 editor
= os
.environ
.get("P4EDITOR")
1744 editor
= read_pipe("git var GIT_EDITOR").strip()
1745 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1747 # If the file was not saved, prompt to see if this patch should
1748 # be skipped. But skip this verification step if configured so.
1749 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1752 # modification time updated means user saved the file
1753 if os
.stat(template_file
).st_mtime
> mtime
:
1757 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1763 def get_diff_description(self
, editedFiles
, filesToAdd
, symlinks
):
1765 if "P4DIFF" in os
.environ
:
1766 del(os
.environ
["P4DIFF"])
1768 for editedFile
in editedFiles
:
1769 diff
+= p4_read_pipe(['diff', '-du',
1770 wildcard_encode(editedFile
)])
1774 for newFile
in filesToAdd
:
1775 newdiff
+= "==== new file ====\n"
1776 newdiff
+= "--- /dev/null\n"
1777 newdiff
+= "+++ %s\n" % newFile
1779 is_link
= os
.path
.islink(newFile
)
1780 expect_link
= newFile
in symlinks
1782 if is_link
and expect_link
:
1783 newdiff
+= "+%s\n" % os
.readlink(newFile
)
1785 f
= open(newFile
, "r")
1786 for line
in f
.readlines():
1787 newdiff
+= "+" + line
1790 return (diff
+ newdiff
).replace('\r\n', '\n')
1792 def applyCommit(self
, id):
1793 """Apply one commit, return True if it succeeded."""
1795 print "Applying", read_pipe(["git", "show", "-s",
1796 "--format=format:%h %s", id])
1798 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1800 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1802 filesToChangeType
= set()
1803 filesToDelete
= set()
1805 pureRenameCopy
= set()
1807 filesToChangeExecBit
= {}
1811 diff
= parseDiffTreeEntry(line
)
1812 modifier
= diff
['status']
1814 all_files
.append(path
)
1818 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1819 filesToChangeExecBit
[path
] = diff
['dst_mode']
1820 editedFiles
.add(path
)
1821 elif modifier
== "A":
1822 filesToAdd
.add(path
)
1823 filesToChangeExecBit
[path
] = diff
['dst_mode']
1824 if path
in filesToDelete
:
1825 filesToDelete
.remove(path
)
1827 dst_mode
= int(diff
['dst_mode'], 8)
1828 if dst_mode
== 0120000:
1831 elif modifier
== "D":
1832 filesToDelete
.add(path
)
1833 if path
in filesToAdd
:
1834 filesToAdd
.remove(path
)
1835 elif modifier
== "C":
1836 src
, dest
= diff
['src'], diff
['dst']
1837 p4_integrate(src
, dest
)
1838 pureRenameCopy
.add(dest
)
1839 if diff
['src_sha1'] != diff
['dst_sha1']:
1841 pureRenameCopy
.discard(dest
)
1842 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1844 pureRenameCopy
.discard(dest
)
1845 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1847 # turn off read-only attribute
1848 os
.chmod(dest
, stat
.S_IWRITE
)
1850 editedFiles
.add(dest
)
1851 elif modifier
== "R":
1852 src
, dest
= diff
['src'], diff
['dst']
1853 if self
.p4HasMoveCommand
:
1854 p4_edit(src
) # src must be open before move
1855 p4_move(src
, dest
) # opens for (move/delete, move/add)
1857 p4_integrate(src
, dest
)
1858 if diff
['src_sha1'] != diff
['dst_sha1']:
1861 pureRenameCopy
.add(dest
)
1862 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1863 if not self
.p4HasMoveCommand
:
1864 p4_edit(dest
) # with move: already open, writable
1865 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1866 if not self
.p4HasMoveCommand
:
1868 os
.chmod(dest
, stat
.S_IWRITE
)
1870 filesToDelete
.add(src
)
1871 editedFiles
.add(dest
)
1872 elif modifier
== "T":
1873 filesToChangeType
.add(path
)
1875 die("unknown modifier %s for %s" % (modifier
, path
))
1877 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1878 patchcmd
= diffcmd
+ " | git apply "
1879 tryPatchCmd
= patchcmd
+ "--check -"
1880 applyPatchCmd
= patchcmd
+ "--check --apply -"
1881 patch_succeeded
= True
1883 if os
.system(tryPatchCmd
) != 0:
1884 fixed_rcs_keywords
= False
1885 patch_succeeded
= False
1886 print "Unfortunately applying the change failed!"
1888 # Patch failed, maybe it's just RCS keyword woes. Look through
1889 # the patch to see if that's possible.
1890 if gitConfigBool("git-p4.attemptRCSCleanup"):
1894 for file in editedFiles | filesToDelete
:
1895 # did this file's delta contain RCS keywords?
1896 pattern
= p4_keywords_regexp_for_file(file)
1899 # this file is a possibility...look for RCS keywords.
1900 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1901 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1902 if regexp
.search(line
):
1904 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1905 kwfiles
[file] = pattern
1908 for file in kwfiles
:
1910 print "zapping %s with %s" % (line
,pattern
)
1911 # File is being deleted, so not open in p4. Must
1912 # disable the read-only bit on windows.
1913 if self
.isWindows
and file not in editedFiles
:
1914 os
.chmod(file, stat
.S_IWRITE
)
1915 self
.patchRCSKeywords(file, kwfiles
[file])
1916 fixed_rcs_keywords
= True
1918 if fixed_rcs_keywords
:
1919 print "Retrying the patch with RCS keywords cleaned up"
1920 if os
.system(tryPatchCmd
) == 0:
1921 patch_succeeded
= True
1923 if not patch_succeeded
:
1924 for f
in editedFiles
:
1929 # Apply the patch for real, and do add/delete/+x handling.
1931 system(applyPatchCmd
)
1933 for f
in filesToChangeType
:
1934 p4_edit(f
, "-t", "auto")
1935 for f
in filesToAdd
:
1937 for f
in filesToDelete
:
1941 # Set/clear executable bits
1942 for f
in filesToChangeExecBit
.keys():
1943 mode
= filesToChangeExecBit
[f
]
1944 setP4ExecBit(f
, mode
)
1947 if len(self
.update_shelve
) > 0:
1948 update_shelve
= self
.update_shelve
.pop(0)
1949 p4_reopen_in_change(update_shelve
, all_files
)
1952 # Build p4 change description, starting with the contents
1953 # of the git commit message.
1955 logMessage
= extractLogMessageFromGitCommit(id)
1956 logMessage
= logMessage
.strip()
1957 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1959 template
= self
.prepareSubmitTemplate(update_shelve
)
1960 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1962 if self
.preserveUser
:
1963 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1965 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1966 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1967 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1968 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1970 separatorLine
= "######## everything below this line is just the diff #######\n"
1971 if not self
.prepare_p4_only
:
1972 submitTemplate
+= separatorLine
1973 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
, symlinks
)
1975 (handle
, fileName
) = tempfile
.mkstemp()
1976 tmpFile
= os
.fdopen(handle
, "w+b")
1978 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1979 tmpFile
.write(submitTemplate
)
1982 if self
.prepare_p4_only
:
1984 # Leave the p4 tree prepared, and the submit template around
1985 # and let the user decide what to do next
1988 print "P4 workspace prepared for submission."
1989 print "To submit or revert, go to client workspace"
1990 print " " + self
.clientPath
1992 print "To submit, use \"p4 submit\" to write a new description,"
1993 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1994 " \"git p4\"." % fileName
1995 print "You can delete the file \"%s\" when finished." % fileName
1997 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1998 print "To preserve change ownership by user %s, you must\n" \
1999 "do \"p4 change -f <change>\" after submitting and\n" \
2000 "edit the User field."
2002 print "After submitting, renamed files must be re-synced."
2003 print "Invoke \"p4 sync -f\" on each of these files:"
2004 for f
in pureRenameCopy
:
2008 print "To revert the changes, use \"p4 revert ...\", and delete"
2009 print "the submit template file \"%s\"" % fileName
2011 print "Since the commit adds new files, they must be deleted:"
2012 for f
in filesToAdd
:
2018 # Let the user edit the change description, then submit it.
2023 if self
.edit_template(fileName
):
2024 # read the edited message and submit
2025 tmpFile
= open(fileName
, "rb")
2026 message
= tmpFile
.read()
2029 message
= message
.replace("\r\n", "\n")
2030 submitTemplate
= message
[:message
.index(separatorLine
)]
2033 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
2035 p4_write_pipe(['shelve', '-i'], submitTemplate
)
2037 p4_write_pipe(['submit', '-i'], submitTemplate
)
2038 # The rename/copy happened by applying a patch that created a
2039 # new file. This leaves it writable, which confuses p4.
2040 for f
in pureRenameCopy
:
2043 if self
.preserveUser
:
2045 # Get last changelist number. Cannot easily get it from
2046 # the submit command output as the output is
2048 changelist
= self
.lastP4Changelist()
2049 self
.modifyChangelistUser(changelist
, p4User
)
2055 if not submitted
or self
.shelve
:
2057 print ("Reverting shelved files.")
2059 print ("Submission cancelled, undoing p4 changes.")
2060 for f
in editedFiles | filesToDelete
:
2062 for f
in filesToAdd
:
2069 # Export git tags as p4 labels. Create a p4 label and then tag
2071 def exportGitTags(self
, gitTags
):
2072 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
2073 if len(validLabelRegexp
) == 0:
2074 validLabelRegexp
= defaultLabelRegexp
2075 m
= re
.compile(validLabelRegexp
)
2077 for name
in gitTags
:
2079 if not m
.match(name
):
2081 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
2084 # Get the p4 commit this corresponds to
2085 logMessage
= extractLogMessageFromGitCommit(name
)
2086 values
= extractSettingsGitLog(logMessage
)
2088 if 'change' not in values
:
2089 # a tag pointing to something not sent to p4; ignore
2091 print "git tag %s does not give a p4 commit" % name
2094 changelist
= values
['change']
2096 # Get the tag details.
2100 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
2103 if re
.match(r
'tag\s+', l
):
2105 elif re
.match(r
'\s*$', l
):
2112 body
= ["lightweight tag imported by git p4\n"]
2114 # Create the label - use the same view as the client spec we are using
2115 clientSpec
= getClientSpec()
2117 labelTemplate
= "Label: %s\n" % name
2118 labelTemplate
+= "Description:\n"
2120 labelTemplate
+= "\t" + b
+ "\n"
2121 labelTemplate
+= "View:\n"
2122 for depot_side
in clientSpec
.mappings
:
2123 labelTemplate
+= "\t%s\n" % depot_side
2126 print "Would create p4 label %s for tag" % name
2127 elif self
.prepare_p4_only
:
2128 print "Not creating p4 label %s for tag due to option" \
2129 " --prepare-p4-only" % name
2131 p4_write_pipe(["label", "-i"], labelTemplate
)
2134 p4_system(["tag", "-l", name
] +
2135 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
2138 print "created p4 label for tag %s" % name
2140 def run(self
, args
):
2142 self
.master
= currentGitBranch()
2143 elif len(args
) == 1:
2144 self
.master
= args
[0]
2145 if not branchExists(self
.master
):
2146 die("Branch %s does not exist" % self
.master
)
2150 for i
in self
.update_shelve
:
2152 sys
.exit("invalid changelist %d" % i
)
2155 allowSubmit
= gitConfig("git-p4.allowSubmit")
2156 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
2157 die("%s is not in git-p4.allowSubmit" % self
.master
)
2159 [upstream
, settings
] = findUpstreamBranchPoint()
2160 self
.depotPath
= settings
['depot-paths'][0]
2161 if len(self
.origin
) == 0:
2162 self
.origin
= upstream
2164 if len(self
.update_shelve
) > 0:
2167 if self
.preserveUser
:
2168 if not self
.canChangeChangelists():
2169 die("Cannot preserve user names without p4 super-user or admin permissions")
2171 # if not set from the command line, try the config file
2172 if self
.conflict_behavior
is None:
2173 val
= gitConfig("git-p4.conflict")
2175 if val
not in self
.conflict_behavior_choices
:
2176 die("Invalid value '%s' for config git-p4.conflict" % val
)
2179 self
.conflict_behavior
= val
2182 print "Origin branch is " + self
.origin
2184 if len(self
.depotPath
) == 0:
2185 print "Internal error: cannot locate perforce depot path from existing branches"
2188 self
.useClientSpec
= False
2189 if gitConfigBool("git-p4.useclientspec"):
2190 self
.useClientSpec
= True
2191 if self
.useClientSpec
:
2192 self
.clientSpecDirs
= getClientSpec()
2194 # Check for the existence of P4 branches
2195 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
2197 if self
.useClientSpec
and not branchesDetected
:
2198 # all files are relative to the client spec
2199 self
.clientPath
= getClientRoot()
2201 self
.clientPath
= p4Where(self
.depotPath
)
2203 if self
.clientPath
== "":
2204 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
2206 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
2207 self
.oldWorkingDirectory
= os
.getcwd()
2209 # ensure the clientPath exists
2210 new_client_dir
= False
2211 if not os
.path
.exists(self
.clientPath
):
2212 new_client_dir
= True
2213 os
.makedirs(self
.clientPath
)
2215 chdir(self
.clientPath
, is_client_path
=True)
2217 print "Would synchronize p4 checkout in %s" % self
.clientPath
2219 print "Synchronizing p4 checkout..."
2221 # old one was destroyed, and maybe nobody told p4
2222 p4_sync("...", "-f")
2229 committish
= self
.master
2233 if self
.commit
!= "":
2234 if self
.commit
.find("..") != -1:
2235 limits_ish
= self
.commit
.split("..")
2236 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish
[0], limits_ish
[1])]):
2237 commits
.append(line
.strip())
2240 commits
.append(self
.commit
)
2242 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, committish
)]):
2243 commits
.append(line
.strip())
2246 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2247 self
.checkAuthorship
= False
2249 self
.checkAuthorship
= True
2251 if self
.preserveUser
:
2252 self
.checkValidP4Users(commits
)
2255 # Build up a set of options to be passed to diff when
2256 # submitting each commit to p4.
2258 if self
.detectRenames
:
2259 # command-line -M arg
2260 self
.diffOpts
= "-M"
2262 # If not explicitly set check the config variable
2263 detectRenames
= gitConfig("git-p4.detectRenames")
2265 if detectRenames
.lower() == "false" or detectRenames
== "":
2267 elif detectRenames
.lower() == "true":
2268 self
.diffOpts
= "-M"
2270 self
.diffOpts
= "-M%s" % detectRenames
2272 # no command-line arg for -C or --find-copies-harder, just
2274 detectCopies
= gitConfig("git-p4.detectCopies")
2275 if detectCopies
.lower() == "false" or detectCopies
== "":
2277 elif detectCopies
.lower() == "true":
2278 self
.diffOpts
+= " -C"
2280 self
.diffOpts
+= " -C%s" % detectCopies
2282 if gitConfigBool("git-p4.detectCopiesHarder"):
2283 self
.diffOpts
+= " --find-copies-harder"
2285 num_shelves
= len(self
.update_shelve
)
2286 if num_shelves
> 0 and num_shelves
!= len(commits
):
2287 sys
.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2288 (len(commits
), num_shelves
))
2291 # Apply the commits, one at a time. On failure, ask if should
2292 # continue to try the rest of the patches, or quit.
2297 last
= len(commits
) - 1
2298 for i
, commit
in enumerate(commits
):
2300 print " ", read_pipe(["git", "show", "-s",
2301 "--format=format:%h %s", commit
])
2304 ok
= self
.applyCommit(commit
)
2306 applied
.append(commit
)
2308 if self
.prepare_p4_only
and i
< last
:
2309 print "Processing only the first commit due to option" \
2310 " --prepare-p4-only"
2315 # prompt for what to do, or use the option/variable
2316 if self
.conflict_behavior
== "ask":
2317 print "What do you want to do?"
2318 response
= raw_input("[s]kip this commit but apply"
2319 " the rest, or [q]uit? ")
2322 elif self
.conflict_behavior
== "skip":
2324 elif self
.conflict_behavior
== "quit":
2327 die("Unknown conflict_behavior '%s'" %
2328 self
.conflict_behavior
)
2330 if response
[0] == "s":
2331 print "Skipping this commit, but applying the rest"
2333 if response
[0] == "q":
2340 chdir(self
.oldWorkingDirectory
)
2341 shelved_applied
= "shelved" if self
.shelve
else "applied"
2344 elif self
.prepare_p4_only
:
2346 elif len(commits
) == len(applied
):
2347 print ("All commits {0}!".format(shelved_applied
))
2351 sync
.branch
= self
.branch
2352 if self
.disable_p4sync
:
2353 sync
.sync_origin_only()
2357 if not self
.disable_rebase
:
2362 if len(applied
) == 0:
2363 print ("No commits {0}.".format(shelved_applied
))
2365 print ("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2371 print star
, read_pipe(["git", "show", "-s",
2372 "--format=format:%h %s", c
])
2373 print "You will have to do 'git p4 sync' and rebase."
2375 if gitConfigBool("git-p4.exportLabels"):
2376 self
.exportLabels
= True
2378 if self
.exportLabels
:
2379 p4Labels
= getP4Labels(self
.depotPath
)
2380 gitTags
= getGitTags()
2382 missingGitTags
= gitTags
- p4Labels
2383 self
.exportGitTags(missingGitTags
)
2385 # exit with error unless everything applied perfectly
2386 if len(commits
) != len(applied
):
2392 """Represent a p4 view ("p4 help views"), and map files in a
2393 repo according to the view."""
2395 def __init__(self
, client_name
):
2397 self
.client_prefix
= "//%s/" % client_name
2398 # cache results of "p4 where" to lookup client file locations
2399 self
.client_spec_path_cache
= {}
2401 def append(self
, view_line
):
2402 """Parse a view line, splitting it into depot and client
2403 sides. Append to self.mappings, preserving order. This
2404 is only needed for tag creation."""
2406 # Split the view line into exactly two words. P4 enforces
2407 # structure on these lines that simplifies this quite a bit.
2409 # Either or both words may be double-quoted.
2410 # Single quotes do not matter.
2411 # Double-quote marks cannot occur inside the words.
2412 # A + or - prefix is also inside the quotes.
2413 # There are no quotes unless they contain a space.
2414 # The line is already white-space stripped.
2415 # The two words are separated by a single space.
2417 if view_line
[0] == '"':
2418 # First word is double quoted. Find its end.
2419 close_quote_index
= view_line
.find('"', 1)
2420 if close_quote_index
<= 0:
2421 die("No first-word closing quote found: %s" % view_line
)
2422 depot_side
= view_line
[1:close_quote_index
]
2423 # skip closing quote and space
2424 rhs_index
= close_quote_index
+ 1 + 1
2426 space_index
= view_line
.find(" ")
2427 if space_index
<= 0:
2428 die("No word-splitting space found: %s" % view_line
)
2429 depot_side
= view_line
[0:space_index
]
2430 rhs_index
= space_index
+ 1
2432 # prefix + means overlay on previous mapping
2433 if depot_side
.startswith("+"):
2434 depot_side
= depot_side
[1:]
2436 # prefix - means exclude this path, leave out of mappings
2438 if depot_side
.startswith("-"):
2440 depot_side
= depot_side
[1:]
2443 self
.mappings
.append(depot_side
)
2445 def convert_client_path(self
, clientFile
):
2446 # chop off //client/ part to make it relative
2447 if not clientFile
.startswith(self
.client_prefix
):
2448 die("No prefix '%s' on clientFile '%s'" %
2449 (self
.client_prefix
, clientFile
))
2450 return clientFile
[len(self
.client_prefix
):]
2452 def update_client_spec_path_cache(self
, files
):
2453 """ Caching file paths by "p4 where" batch query """
2455 # List depot file paths exclude that already cached
2456 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2458 if len(fileArgs
) == 0:
2459 return # All files in cache
2461 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2462 for res
in where_result
:
2463 if "code" in res
and res
["code"] == "error":
2464 # assume error is "... file(s) not in client view"
2466 if "clientFile" not in res
:
2467 die("No clientFile in 'p4 where' output")
2469 # it will list all of them, but only one not unmap-ped
2471 if gitConfigBool("core.ignorecase"):
2472 res
['depotFile'] = res
['depotFile'].lower()
2473 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2475 # not found files or unmap files set to ""
2476 for depotFile
in fileArgs
:
2477 if gitConfigBool("core.ignorecase"):
2478 depotFile
= depotFile
.lower()
2479 if depotFile
not in self
.client_spec_path_cache
:
2480 self
.client_spec_path_cache
[depotFile
] = ""
2482 def map_in_client(self
, depot_path
):
2483 """Return the relative location in the client where this
2484 depot file should live. Returns "" if the file should
2485 not be mapped in the client."""
2487 if gitConfigBool("core.ignorecase"):
2488 depot_path
= depot_path
.lower()
2490 if depot_path
in self
.client_spec_path_cache
:
2491 return self
.client_spec_path_cache
[depot_path
]
2493 die( "Error: %s is not found in client spec path" % depot_path
)
2496 class P4Sync(Command
, P4UserMap
):
2497 delete_actions
= ( "delete", "move/delete", "purge" )
2500 Command
.__init
__(self
)
2501 P4UserMap
.__init
__(self
)
2503 optparse
.make_option("--branch", dest
="branch"),
2504 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2505 optparse
.make_option("--changesfile", dest
="changesFile"),
2506 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2507 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2508 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2509 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2510 help="Import into refs/heads/ , not refs/remotes"),
2511 optparse
.make_option("--max-changes", dest
="maxChanges",
2512 help="Maximum number of changes to import"),
2513 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2514 help="Internal block size to use when iteratively calling p4 changes"),
2515 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2516 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2517 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2518 help="Only sync files that are included in the Perforce Client Spec"),
2519 optparse
.make_option("-/", dest
="cloneExclude",
2520 action
="append", type="string",
2521 help="exclude depot path"),
2523 self
.description
= """Imports from Perforce into a git repository.\n
2525 //depot/my/project/ -- to import the current head
2526 //depot/my/project/@all -- to import everything
2527 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2529 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2531 self
.usage
+= " //depot/path[@revRange]"
2533 self
.createdBranches
= set()
2534 self
.committedChanges
= set()
2536 self
.detectBranches
= False
2537 self
.detectLabels
= False
2538 self
.importLabels
= False
2539 self
.changesFile
= ""
2540 self
.syncWithOrigin
= True
2541 self
.importIntoRemotes
= True
2542 self
.maxChanges
= ""
2543 self
.changes_block_size
= None
2544 self
.keepRepoPath
= False
2545 self
.depotPaths
= None
2546 self
.p4BranchesInGit
= []
2547 self
.cloneExclude
= []
2548 self
.useClientSpec
= False
2549 self
.useClientSpec_from_options
= False
2550 self
.clientSpecDirs
= None
2551 self
.tempBranches
= []
2552 self
.tempBranchLocation
= "refs/git-p4-tmp"
2553 self
.largeFileSystem
= None
2554 self
.suppress_meta_comment
= False
2556 if gitConfig('git-p4.largeFileSystem'):
2557 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2558 self
.largeFileSystem
= largeFileSystemConstructor(
2559 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2562 if gitConfig("git-p4.syncFromOrigin") == "false":
2563 self
.syncWithOrigin
= False
2565 self
.depotPaths
= []
2566 self
.changeRange
= ""
2567 self
.previousDepotPaths
= []
2568 self
.hasOrigin
= False
2570 # map from branch depot path to parent branch
2571 self
.knownBranches
= {}
2572 self
.initialParents
= {}
2574 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2577 # Force a checkpoint in fast-import and wait for it to finish
2578 def checkpoint(self
):
2579 self
.gitStream
.write("checkpoint\n\n")
2580 self
.gitStream
.write("progress checkpoint\n\n")
2581 out
= self
.gitOutput
.readline()
2583 print "checkpoint finished: " + out
2585 def cmp_shelved(self
, path
, filerev
, revision
):
2586 """ Determine if a path at revision #filerev is the same as the file
2587 at revision @revision for a shelved changelist. If they don't match,
2588 unshelving won't be safe (we will get other changes mixed in).
2590 This is comparing the revision that the shelved changelist is *based* on, not
2591 the shelved changelist itself.
2593 ret
= p4Cmd(["diff2", "{0}#{1}".format(path
, filerev
), "{0}@{1}".format(path
, revision
)])
2595 print("p4 diff2 path %s filerev %s revision %s => %s" % (path
, filerev
, revision
, ret
))
2596 return ret
["status"] == "identical"
2598 def extractFilesFromCommit(self
, commit
, shelved
=False, shelved_cl
= 0, origin_revision
= 0):
2599 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2600 for path
in self
.cloneExclude
]
2603 while "depotFile%s" % fnum
in commit
:
2604 path
= commit
["depotFile%s" % fnum
]
2606 if [p
for p
in self
.cloneExclude
2607 if p4PathStartsWith(path
, p
)]:
2610 found
= [p
for p
in self
.depotPaths
2611 if p4PathStartsWith(path
, p
)]
2618 file["rev"] = commit
["rev%s" % fnum
]
2619 file["action"] = commit
["action%s" % fnum
]
2620 file["type"] = commit
["type%s" % fnum
]
2622 file["shelved_cl"] = int(shelved_cl
)
2624 # For shelved changelists, check that the revision of each file that the
2625 # shelve was based on matches the revision that we are using for the
2626 # starting point for git-fast-import (self.initialParent). Otherwise
2627 # the resulting diff will contain deltas from multiple commits.
2629 if file["action"] != "add" and \
2630 not self
.cmp_shelved(path
, file["rev"], origin_revision
):
2631 sys
.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2632 commit
["change"], self
.initialParent
, path
))
2638 def extractJobsFromCommit(self
, commit
):
2641 while "job%s" % jnum
in commit
:
2642 job
= commit
["job%s" % jnum
]
2647 def stripRepoPath(self
, path
, prefixes
):
2648 """When streaming files, this is called to map a p4 depot path
2649 to where it should go in git. The prefixes are either
2650 self.depotPaths, or self.branchPrefixes in the case of
2651 branch detection."""
2653 if self
.useClientSpec
:
2654 # branch detection moves files up a level (the branch name)
2655 # from what client spec interpretation gives
2656 path
= self
.clientSpecDirs
.map_in_client(path
)
2657 if self
.detectBranches
:
2658 for b
in self
.knownBranches
:
2659 if path
.startswith(b
+ "/"):
2660 path
= path
[len(b
)+1:]
2662 elif self
.keepRepoPath
:
2663 # Preserve everything in relative path name except leading
2664 # //depot/; just look at first prefix as they all should
2665 # be in the same depot.
2666 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2667 if p4PathStartsWith(path
, depot
):
2668 path
= path
[len(depot
):]
2672 if p4PathStartsWith(path
, p
):
2673 path
= path
[len(p
):]
2676 path
= wildcard_decode(path
)
2679 def splitFilesIntoBranches(self
, commit
):
2680 """Look at each depotFile in the commit to figure out to what
2681 branch it belongs."""
2683 if self
.clientSpecDirs
:
2684 files
= self
.extractFilesFromCommit(commit
)
2685 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2689 while "depotFile%s" % fnum
in commit
:
2690 path
= commit
["depotFile%s" % fnum
]
2691 found
= [p
for p
in self
.depotPaths
2692 if p4PathStartsWith(path
, p
)]
2699 file["rev"] = commit
["rev%s" % fnum
]
2700 file["action"] = commit
["action%s" % fnum
]
2701 file["type"] = commit
["type%s" % fnum
]
2704 # start with the full relative path where this file would
2706 if self
.useClientSpec
:
2707 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2709 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2711 for branch
in self
.knownBranches
.keys():
2712 # add a trailing slash so that a commit into qt/4.2foo
2713 # doesn't end up in qt/4.2, e.g.
2714 if relPath
.startswith(branch
+ "/"):
2715 if branch
not in branches
:
2716 branches
[branch
] = []
2717 branches
[branch
].append(file)
2722 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2723 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2724 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2726 self
.gitStream
.write(d
)
2727 self
.gitStream
.write('\n')
2729 def encodeWithUTF8(self
, path
):
2731 path
.decode('ascii')
2734 if gitConfig('git-p4.pathEncoding'):
2735 encoding
= gitConfig('git-p4.pathEncoding')
2736 path
= path
.decode(encoding
, 'replace').encode('utf8', 'replace')
2738 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, path
)
2741 # output one file from the P4 stream
2742 # - helper for streamP4Files
2744 def streamOneP4File(self
, file, contents
):
2745 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2746 relPath
= self
.encodeWithUTF8(relPath
)
2748 size
= int(self
.stream_file
['fileSize'])
2749 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2752 (type_base
, type_mods
) = split_p4_type(file["type"])
2755 if "x" in type_mods
:
2757 if type_base
== "symlink":
2759 # p4 print on a symlink sometimes contains "target\n";
2760 # if it does, remove the newline
2761 data
= ''.join(contents
)
2763 # Some version of p4 allowed creating a symlink that pointed
2764 # to nothing. This causes p4 errors when checking out such
2765 # a change, and errors here too. Work around it by ignoring
2766 # the bad symlink; hopefully a future change fixes it.
2767 print "\nIgnoring empty symlink in %s" % file['depotFile']
2769 elif data
[-1] == '\n':
2770 contents
= [data
[:-1]]
2774 if type_base
== "utf16":
2775 # p4 delivers different text in the python output to -G
2776 # than it does when using "print -o", or normal p4 client
2777 # operations. utf16 is converted to ascii or utf8, perhaps.
2778 # But ascii text saved as -t utf16 is completely mangled.
2779 # Invoke print -o to get the real contents.
2781 # On windows, the newlines will always be mangled by print, so put
2782 # them back too. This is not needed to the cygwin windows version,
2783 # just the native "NT" type.
2786 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2787 except Exception as e
:
2788 if 'Translation of file content failed' in str(e
):
2789 type_base
= 'binary'
2793 if p4_version_string().find('/NT') >= 0:
2794 text
= text
.replace('\r\n', '\n')
2797 if type_base
== "apple":
2798 # Apple filetype files will be streamed as a concatenation of
2799 # its appledouble header and the contents. This is useless
2800 # on both macs and non-macs. If using "print -q -o xx", it
2801 # will create "xx" with the data, and "%xx" with the header.
2802 # This is also not very useful.
2804 # Ideally, someday, this script can learn how to generate
2805 # appledouble files directly and import those to git, but
2806 # non-mac machines can never find a use for apple filetype.
2807 print "\nIgnoring apple filetype file %s" % file['depotFile']
2810 # Note that we do not try to de-mangle keywords on utf16 files,
2811 # even though in theory somebody may want that.
2812 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2814 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2815 text
= ''.join(contents
)
2816 text
= regexp
.sub(r
'$\1$', text
)
2819 if self
.largeFileSystem
:
2820 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2822 self
.writeToGitStream(git_mode
, relPath
, contents
)
2824 def streamOneP4Deletion(self
, file):
2825 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2826 relPath
= self
.encodeWithUTF8(relPath
)
2828 sys
.stdout
.write("delete %s\n" % relPath
)
2830 self
.gitStream
.write("D %s\n" % relPath
)
2832 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2833 self
.largeFileSystem
.removeLargeFile(relPath
)
2835 # handle another chunk of streaming data
2836 def streamP4FilesCb(self
, marshalled
):
2838 # catch p4 errors and complain
2840 if "code" in marshalled
:
2841 if marshalled
["code"] == "error":
2842 if "data" in marshalled
:
2843 err
= marshalled
["data"].rstrip()
2845 if not err
and 'fileSize' in self
.stream_file
:
2846 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2847 if required_bytes
> 0:
2848 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2849 os
.getcwd(), required_bytes
/1024/1024
2854 if self
.stream_have_file_info
:
2855 if "depotFile" in self
.stream_file
:
2856 f
= self
.stream_file
["depotFile"]
2857 # force a failure in fast-import, else an empty
2858 # commit will be made
2859 self
.gitStream
.write("\n")
2860 self
.gitStream
.write("die-now\n")
2861 self
.gitStream
.close()
2862 # ignore errors, but make sure it exits first
2863 self
.importProcess
.wait()
2865 die("Error from p4 print for %s: %s" % (f
, err
))
2867 die("Error from p4 print: %s" % err
)
2869 if 'depotFile' in marshalled
and self
.stream_have_file_info
:
2870 # start of a new file - output the old one first
2871 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2872 self
.stream_file
= {}
2873 self
.stream_contents
= []
2874 self
.stream_have_file_info
= False
2876 # pick up the new file information... for the
2877 # 'data' field we need to append to our array
2878 for k
in marshalled
.keys():
2880 if 'streamContentSize' not in self
.stream_file
:
2881 self
.stream_file
['streamContentSize'] = 0
2882 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2883 self
.stream_contents
.append(marshalled
['data'])
2885 self
.stream_file
[k
] = marshalled
[k
]
2888 'streamContentSize' in self
.stream_file
and
2889 'fileSize' in self
.stream_file
and
2890 'depotFile' in self
.stream_file
):
2891 size
= int(self
.stream_file
["fileSize"])
2893 progress
= 100*self
.stream_file
['streamContentSize']/size
2894 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2897 self
.stream_have_file_info
= True
2899 # Stream directly from "p4 files" into "git fast-import"
2900 def streamP4Files(self
, files
):
2906 filesForCommit
.append(f
)
2907 if f
['action'] in self
.delete_actions
:
2908 filesToDelete
.append(f
)
2910 filesToRead
.append(f
)
2913 for f
in filesToDelete
:
2914 self
.streamOneP4Deletion(f
)
2916 if len(filesToRead
) > 0:
2917 self
.stream_file
= {}
2918 self
.stream_contents
= []
2919 self
.stream_have_file_info
= False
2921 # curry self argument
2922 def streamP4FilesCbSelf(entry
):
2923 self
.streamP4FilesCb(entry
)
2926 for f
in filesToRead
:
2927 if 'shelved_cl' in f
:
2928 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2930 fileArg
= '%s@=%d' % (f
['path'], f
['shelved_cl'])
2932 fileArg
= '%s#%s' % (f
['path'], f
['rev'])
2934 fileArgs
.append(fileArg
)
2936 p4CmdList(["-x", "-", "print"],
2938 cb
=streamP4FilesCbSelf
)
2941 if 'depotFile' in self
.stream_file
:
2942 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2944 def make_email(self
, userid
):
2945 if userid
in self
.users
:
2946 return self
.users
[userid
]
2948 return "%s <a@b>" % userid
2950 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2951 """ Stream a p4 tag.
2952 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2956 print "writing tag %s for commit %s" % (labelName
, commit
)
2957 gitStream
.write("tag %s\n" % labelName
)
2958 gitStream
.write("from %s\n" % commit
)
2960 if 'Owner' in labelDetails
:
2961 owner
= labelDetails
["Owner"]
2965 # Try to use the owner of the p4 label, or failing that,
2966 # the current p4 user id.
2968 email
= self
.make_email(owner
)
2970 email
= self
.make_email(self
.p4UserId())
2971 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2973 gitStream
.write("tagger %s\n" % tagger
)
2975 print "labelDetails=",labelDetails
2976 if 'Description' in labelDetails
:
2977 description
= labelDetails
['Description']
2979 description
= 'Label from git p4'
2981 gitStream
.write("data %d\n" % len(description
))
2982 gitStream
.write(description
)
2983 gitStream
.write("\n")
2985 def inClientSpec(self
, path
):
2986 if not self
.clientSpecDirs
:
2988 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
2989 if not inClientSpec
and self
.verbose
:
2990 print('Ignoring file outside of client spec: {0}'.format(path
))
2993 def hasBranchPrefix(self
, path
):
2994 if not self
.branchPrefixes
:
2996 hasPrefix
= [p
for p
in self
.branchPrefixes
2997 if p4PathStartsWith(path
, p
)]
2998 if not hasPrefix
and self
.verbose
:
2999 print('Ignoring file outside of prefix: {0}'.format(path
))
3002 def commit(self
, details
, files
, branch
, parent
= ""):
3003 epoch
= details
["time"]
3004 author
= details
["user"]
3005 jobs
= self
.extractJobsFromCommit(details
)
3008 print('commit into {0}'.format(branch
))
3010 if self
.clientSpecDirs
:
3011 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
3013 files
= [f
for f
in files
3014 if self
.inClientSpec(f
['path']) and self
.hasBranchPrefix(f
['path'])]
3016 if not files
and not gitConfigBool('git-p4.keepEmptyCommits'):
3017 print('Ignoring revision {0} as it would produce an empty commit.'
3018 .format(details
['change']))
3021 self
.gitStream
.write("commit %s\n" % branch
)
3022 self
.gitStream
.write("mark :%s\n" % details
["change"])
3023 self
.committedChanges
.add(int(details
["change"]))
3025 if author
not in self
.users
:
3026 self
.getUserMapFromPerforceServer()
3027 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
3029 self
.gitStream
.write("committer %s\n" % committer
)
3031 self
.gitStream
.write("data <<EOT\n")
3032 self
.gitStream
.write(details
["desc"])
3034 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
3036 if not self
.suppress_meta_comment
:
3037 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3038 (','.join(self
.branchPrefixes
), details
["change"]))
3039 if len(details
['options']) > 0:
3040 self
.gitStream
.write(": options = %s" % details
['options'])
3041 self
.gitStream
.write("]\n")
3043 self
.gitStream
.write("EOT\n\n")
3047 print "parent %s" % parent
3048 self
.gitStream
.write("from %s\n" % parent
)
3050 self
.streamP4Files(files
)
3051 self
.gitStream
.write("\n")
3053 change
= int(details
["change"])
3055 if change
in self
.labels
:
3056 label
= self
.labels
[change
]
3057 labelDetails
= label
[0]
3058 labelRevisions
= label
[1]
3060 print "Change %s is labelled %s" % (change
, labelDetails
)
3062 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
3063 for p
in self
.branchPrefixes
])
3065 if len(files
) == len(labelRevisions
):
3069 if info
["action"] in self
.delete_actions
:
3071 cleanedFiles
[info
["depotFile"]] = info
["rev"]
3073 if cleanedFiles
== labelRevisions
:
3074 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
3078 print ("Tag %s does not match with change %s: files do not match."
3079 % (labelDetails
["label"], change
))
3083 print ("Tag %s does not match with change %s: file count is different."
3084 % (labelDetails
["label"], change
))
3086 # Build a dictionary of changelists and labels, for "detect-labels" option.
3087 def getLabels(self
):
3090 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
3091 if len(l
) > 0 and not self
.silent
:
3092 print("Finding files belonging to labels in %s" % self
.depotPaths
)
3095 label
= output
["label"]
3099 print "Querying files for label %s" % label
3100 for file in p4CmdList(["files"] +
3101 ["%s...@%s" % (p
, label
)
3102 for p
in self
.depotPaths
]):
3103 revisions
[file["depotFile"]] = file["rev"]
3104 change
= int(file["change"])
3105 if change
> newestChange
:
3106 newestChange
= change
3108 self
.labels
[newestChange
] = [output
, revisions
]
3111 print "Label changes: %s" % self
.labels
.keys()
3113 # Import p4 labels as git tags. A direct mapping does not
3114 # exist, so assume that if all the files are at the same revision
3115 # then we can use that, or it's something more complicated we should
3117 def importP4Labels(self
, stream
, p4Labels
):
3119 print "import p4 labels: " + ' '.join(p4Labels
)
3121 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
3122 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
3123 if len(validLabelRegexp
) == 0:
3124 validLabelRegexp
= defaultLabelRegexp
3125 m
= re
.compile(validLabelRegexp
)
3127 for name
in p4Labels
:
3130 if not m
.match(name
):
3132 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
3135 if name
in ignoredP4Labels
:
3138 labelDetails
= p4CmdList(['label', "-o", name
])[0]
3140 # get the most recent changelist for each file in this label
3141 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
3142 for p
in self
.depotPaths
])
3144 if 'change' in change
:
3145 # find the corresponding git commit; take the oldest commit
3146 changelist
= int(change
['change'])
3147 if changelist
in self
.committedChanges
:
3148 gitCommit
= ":%d" % changelist
# use a fast-import mark
3151 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
3152 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
3153 if len(gitCommit
) == 0:
3154 print "importing label %s: could not find git commit for changelist %d" % (name
, changelist
)
3157 gitCommit
= gitCommit
.strip()
3160 # Convert from p4 time format
3162 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
3164 print "Could not convert label time %s" % labelDetails
['Update']
3167 when
= int(time
.mktime(tmwhen
))
3168 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
3170 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
3173 print "Label %s has no changelists - possibly deleted?" % name
3176 # We can't import this label; don't try again as it will get very
3177 # expensive repeatedly fetching all the files for labels that will
3178 # never be imported. If the label is moved in the future, the
3179 # ignore will need to be removed manually.
3180 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
3182 def guessProjectName(self
):
3183 for p
in self
.depotPaths
:
3186 p
= p
[p
.strip().rfind("/") + 1:]
3187 if not p
.endswith("/"):
3191 def getBranchMapping(self
):
3192 lostAndFoundBranches
= set()
3194 user
= gitConfig("git-p4.branchUser")
3196 command
= "branches -u %s" % user
3198 command
= "branches"
3200 for info
in p4CmdList(command
):
3201 details
= p4Cmd(["branch", "-o", info
["branch"]])
3203 while "View%s" % viewIdx
in details
:
3204 paths
= details
["View%s" % viewIdx
].split(" ")
3205 viewIdx
= viewIdx
+ 1
3206 # require standard //depot/foo/... //depot/bar/... mapping
3207 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
3210 destination
= paths
[1]
3212 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
3213 source
= source
[len(self
.depotPaths
[0]):-4]
3214 destination
= destination
[len(self
.depotPaths
[0]):-4]
3216 if destination
in self
.knownBranches
:
3218 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
3219 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
3222 self
.knownBranches
[destination
] = source
3224 lostAndFoundBranches
.discard(destination
)
3226 if source
not in self
.knownBranches
:
3227 lostAndFoundBranches
.add(source
)
3229 # Perforce does not strictly require branches to be defined, so we also
3230 # check git config for a branch list.
3232 # Example of branch definition in git config file:
3234 # branchList=main:branchA
3235 # branchList=main:branchB
3236 # branchList=branchA:branchC
3237 configBranches
= gitConfigList("git-p4.branchList")
3238 for branch
in configBranches
:
3240 (source
, destination
) = branch
.split(":")
3241 self
.knownBranches
[destination
] = source
3243 lostAndFoundBranches
.discard(destination
)
3245 if source
not in self
.knownBranches
:
3246 lostAndFoundBranches
.add(source
)
3249 for branch
in lostAndFoundBranches
:
3250 self
.knownBranches
[branch
] = branch
3252 def getBranchMappingFromGitBranches(self
):
3253 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3254 for branch
in branches
.keys():
3255 if branch
== "master":
3258 branch
= branch
[len(self
.projectName
):]
3259 self
.knownBranches
[branch
] = branch
3261 def updateOptionDict(self
, d
):
3263 if self
.keepRepoPath
:
3264 option_keys
['keepRepoPath'] = 1
3266 d
["options"] = ' '.join(sorted(option_keys
.keys()))
3268 def readOptions(self
, d
):
3269 self
.keepRepoPath
= ('options' in d
3270 and ('keepRepoPath' in d
['options']))
3272 def gitRefForBranch(self
, branch
):
3273 if branch
== "main":
3274 return self
.refPrefix
+ "master"
3276 if len(branch
) <= 0:
3279 return self
.refPrefix
+ self
.projectName
+ branch
3281 def gitCommitByP4Change(self
, ref
, change
):
3283 print "looking in ref " + ref
+ " for change %s using bisect..." % change
3286 latestCommit
= parseRevision(ref
)
3290 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
3291 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
3296 log
= extractLogMessageFromGitCommit(next
)
3297 settings
= extractSettingsGitLog(log
)
3298 currentChange
= int(settings
['change'])
3300 print "current change %s" % currentChange
3302 if currentChange
== change
:
3304 print "found %s" % next
3307 if currentChange
< change
:
3308 earliestCommit
= "^%s" % next
3310 latestCommit
= "%s" % next
3314 def importNewBranch(self
, branch
, maxChange
):
3315 # make fast-import flush all changes to disk and update the refs using the checkpoint
3316 # command so that we can try to find the branch parent in the git history
3317 self
.gitStream
.write("checkpoint\n\n");
3318 self
.gitStream
.flush();
3319 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3320 range = "@1,%s" % maxChange
3321 #print "prefix" + branchPrefix
3322 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3323 if len(changes
) <= 0:
3325 firstChange
= changes
[0]
3326 #print "first change in branch: %s" % firstChange
3327 sourceBranch
= self
.knownBranches
[branch
]
3328 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3329 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3330 #print "source " + sourceBranch
3332 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3333 #print "branch parent: %s" % branchParentChange
3334 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3335 if len(gitParent
) > 0:
3336 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3337 #print "parent git commit: %s" % gitParent
3339 self
.importChanges(changes
)
3342 def searchParent(self
, parent
, branch
, target
):
3344 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
3345 "--no-merges", parent
]):
3347 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
3350 print "Found parent of %s in commit %s" % (branch
, blob
)
3357 def importChanges(self
, changes
, shelved
=False, origin_revision
=0):
3359 for change
in changes
:
3360 description
= p4_describe(change
, shelved
)
3361 self
.updateOptionDict(description
)
3364 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3369 if self
.detectBranches
:
3370 branches
= self
.splitFilesIntoBranches(description
)
3371 for branch
in branches
.keys():
3373 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3374 self
.branchPrefixes
= [ branchPrefix
]
3378 filesForCommit
= branches
[branch
]
3381 print "branch is %s" % branch
3383 self
.updatedBranches
.add(branch
)
3385 if branch
not in self
.createdBranches
:
3386 self
.createdBranches
.add(branch
)
3387 parent
= self
.knownBranches
[branch
]
3388 if parent
== branch
:
3391 fullBranch
= self
.projectName
+ branch
3392 if fullBranch
not in self
.p4BranchesInGit
:
3394 print("\n Importing new branch %s" % fullBranch
);
3395 if self
.importNewBranch(branch
, change
- 1):
3397 self
.p4BranchesInGit
.append(fullBranch
)
3399 print("\n Resuming with change %s" % change
);
3402 print "parent determined through known branches: %s" % parent
3404 branch
= self
.gitRefForBranch(branch
)
3405 parent
= self
.gitRefForBranch(parent
)
3408 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
3410 if len(parent
) == 0 and branch
in self
.initialParents
:
3411 parent
= self
.initialParents
[branch
]
3412 del self
.initialParents
[branch
]
3416 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3418 print "Creating temporary branch: " + tempBranch
3419 self
.commit(description
, filesForCommit
, tempBranch
)
3420 self
.tempBranches
.append(tempBranch
)
3422 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3424 self
.commit(description
, filesForCommit
, branch
, blob
)
3427 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3428 self
.commit(description
, filesForCommit
, branch
, parent
)
3430 files
= self
.extractFilesFromCommit(description
, shelved
, change
, origin_revision
)
3431 self
.commit(description
, files
, self
.branch
,
3433 # only needed once, to connect to the previous commit
3434 self
.initialParent
= ""
3436 print self
.gitError
.read()
3439 def sync_origin_only(self
):
3440 if self
.syncWithOrigin
:
3441 self
.hasOrigin
= originP4BranchesExist()
3444 print 'Syncing with origin first, using "git fetch origin"'
3445 system("git fetch origin")
3447 def importHeadRevision(self
, revision
):
3448 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3451 details
["user"] = "git perforce import user"
3452 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3453 % (' '.join(self
.depotPaths
), revision
))
3454 details
["change"] = revision
3458 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3460 for info
in p4CmdList(["files"] + fileArgs
):
3462 if 'code' in info
and info
['code'] == 'error':
3463 sys
.stderr
.write("p4 returned an error: %s\n"
3465 if info
['data'].find("must refer to client") >= 0:
3466 sys
.stderr
.write("This particular p4 error is misleading.\n")
3467 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3468 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3470 if 'p4ExitCode' in info
:
3471 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3475 change
= int(info
["change"])
3476 if change
> newestRevision
:
3477 newestRevision
= change
3479 if info
["action"] in self
.delete_actions
:
3480 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3481 #fileCnt = fileCnt + 1
3484 for prop
in ["depotFile", "rev", "action", "type" ]:
3485 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3487 fileCnt
= fileCnt
+ 1
3489 details
["change"] = newestRevision
3491 # Use time from top-most change so that all git p4 clones of
3492 # the same p4 repo have the same commit SHA1s.
3493 res
= p4_describe(newestRevision
)
3494 details
["time"] = res
["time"]
3496 self
.updateOptionDict(details
)
3498 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3500 print "IO error with git fast-import. Is your git version recent enough?"
3501 print self
.gitError
.read()
3503 def openStreams(self
):
3504 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3505 stdin
=subprocess
.PIPE
,
3506 stdout
=subprocess
.PIPE
,
3507 stderr
=subprocess
.PIPE
);
3508 self
.gitOutput
= self
.importProcess
.stdout
3509 self
.gitStream
= self
.importProcess
.stdin
3510 self
.gitError
= self
.importProcess
.stderr
3512 def closeStreams(self
):
3513 self
.gitStream
.close()
3514 if self
.importProcess
.wait() != 0:
3515 die("fast-import failed: %s" % self
.gitError
.read())
3516 self
.gitOutput
.close()
3517 self
.gitError
.close()
3519 def run(self
, args
):
3520 if self
.importIntoRemotes
:
3521 self
.refPrefix
= "refs/remotes/p4/"
3523 self
.refPrefix
= "refs/heads/p4/"
3525 self
.sync_origin_only()
3527 branch_arg_given
= bool(self
.branch
)
3528 if len(self
.branch
) == 0:
3529 self
.branch
= self
.refPrefix
+ "master"
3530 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3531 system("git update-ref %s refs/heads/p4" % self
.branch
)
3532 system("git branch -D p4")
3534 # accept either the command-line option, or the configuration variable
3535 if self
.useClientSpec
:
3536 # will use this after clone to set the variable
3537 self
.useClientSpec_from_options
= True
3539 if gitConfigBool("git-p4.useclientspec"):
3540 self
.useClientSpec
= True
3541 if self
.useClientSpec
:
3542 self
.clientSpecDirs
= getClientSpec()
3544 # TODO: should always look at previous commits,
3545 # merge with previous imports, if possible.
3548 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3550 # branches holds mapping from branch name to sha1
3551 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3553 # restrict to just this one, disabling detect-branches
3554 if branch_arg_given
:
3555 short
= self
.branch
.split("/")[-1]
3556 if short
in branches
:
3557 self
.p4BranchesInGit
= [ short
]
3559 self
.p4BranchesInGit
= branches
.keys()
3561 if len(self
.p4BranchesInGit
) > 1:
3563 print "Importing from/into multiple branches"
3564 self
.detectBranches
= True
3565 for branch
in branches
.keys():
3566 self
.initialParents
[self
.refPrefix
+ branch
] = \
3570 print "branches: %s" % self
.p4BranchesInGit
3573 for branch
in self
.p4BranchesInGit
:
3574 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3576 settings
= extractSettingsGitLog(logMsg
)
3578 self
.readOptions(settings
)
3579 if ('depot-paths' in settings
3580 and 'change' in settings
):
3581 change
= int(settings
['change']) + 1
3582 p4Change
= max(p4Change
, change
)
3584 depotPaths
= sorted(settings
['depot-paths'])
3585 if self
.previousDepotPaths
== []:
3586 self
.previousDepotPaths
= depotPaths
3589 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3590 prev_list
= prev
.split("/")
3591 cur_list
= cur
.split("/")
3592 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3593 if cur_list
[i
] != prev_list
[i
]:
3597 paths
.append ("/".join(cur_list
[:i
+ 1]))
3599 self
.previousDepotPaths
= paths
3602 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3603 self
.changeRange
= "@%s,#head" % p4Change
3604 if not self
.silent
and not self
.detectBranches
:
3605 print "Performing incremental import into %s git branch" % self
.branch
3607 # accept multiple ref name abbreviations:
3608 # refs/foo/bar/branch -> use it exactly
3609 # p4/branch -> prepend refs/remotes/ or refs/heads/
3610 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3611 if not self
.branch
.startswith("refs/"):
3612 if self
.importIntoRemotes
:
3613 prepend
= "refs/remotes/"
3615 prepend
= "refs/heads/"
3616 if not self
.branch
.startswith("p4/"):
3618 self
.branch
= prepend
+ self
.branch
3620 if len(args
) == 0 and self
.depotPaths
:
3622 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3624 if self
.depotPaths
and self
.depotPaths
!= args
:
3625 print ("previous import used depot path %s and now %s was specified. "
3626 "This doesn't work!" % (' '.join (self
.depotPaths
),
3630 self
.depotPaths
= sorted(args
)
3635 # Make sure no revision specifiers are used when --changesfile
3637 bad_changesfile
= False
3638 if len(self
.changesFile
) > 0:
3639 for p
in self
.depotPaths
:
3640 if p
.find("@") >= 0 or p
.find("#") >= 0:
3641 bad_changesfile
= True
3644 die("Option --changesfile is incompatible with revision specifiers")
3647 for p
in self
.depotPaths
:
3648 if p
.find("@") != -1:
3649 atIdx
= p
.index("@")
3650 self
.changeRange
= p
[atIdx
:]
3651 if self
.changeRange
== "@all":
3652 self
.changeRange
= ""
3653 elif ',' not in self
.changeRange
:
3654 revision
= self
.changeRange
3655 self
.changeRange
= ""
3657 elif p
.find("#") != -1:
3658 hashIdx
= p
.index("#")
3659 revision
= p
[hashIdx
:]
3661 elif self
.previousDepotPaths
== []:
3662 # pay attention to changesfile, if given, else import
3663 # the entire p4 tree at the head revision
3664 if len(self
.changesFile
) == 0:
3667 p
= re
.sub ("\.\.\.$", "", p
)
3668 if not p
.endswith("/"):
3673 self
.depotPaths
= newPaths
3675 # --detect-branches may change this for each branch
3676 self
.branchPrefixes
= self
.depotPaths
3678 self
.loadUserMapFromCache()
3680 if self
.detectLabels
:
3683 if self
.detectBranches
:
3684 ## FIXME - what's a P4 projectName ?
3685 self
.projectName
= self
.guessProjectName()
3688 self
.getBranchMappingFromGitBranches()
3690 self
.getBranchMapping()
3692 print "p4-git branches: %s" % self
.p4BranchesInGit
3693 print "initial parents: %s" % self
.initialParents
3694 for b
in self
.p4BranchesInGit
:
3698 b
= b
[len(self
.projectName
):]
3699 self
.createdBranches
.add(b
)
3704 self
.importHeadRevision(revision
)
3708 if len(self
.changesFile
) > 0:
3709 output
= open(self
.changesFile
).readlines()
3712 changeSet
.add(int(line
))
3714 for change
in changeSet
:
3715 changes
.append(change
)
3719 # catch "git p4 sync" with no new branches, in a repo that
3720 # does not have any existing p4 branches
3722 if not self
.p4BranchesInGit
:
3723 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3725 # The default branch is master, unless --branch is used to
3726 # specify something else. Make sure it exists, or complain
3727 # nicely about how to use --branch.
3728 if not self
.detectBranches
:
3729 if not branch_exists(self
.branch
):
3730 if branch_arg_given
:
3731 die("Error: branch %s does not exist." % self
.branch
)
3733 die("Error: no branch %s; perhaps specify one with --branch." %
3737 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3739 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3741 if len(self
.maxChanges
) > 0:
3742 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3744 if len(changes
) == 0:
3746 print "No changes to import!"
3748 if not self
.silent
and not self
.detectBranches
:
3749 print "Import destination: %s" % self
.branch
3751 self
.updatedBranches
= set()
3753 if not self
.detectBranches
:
3755 # start a new branch
3756 self
.initialParent
= ""
3758 # build on a previous revision
3759 self
.initialParent
= parseRevision(self
.branch
)
3761 self
.importChanges(changes
)
3765 if len(self
.updatedBranches
) > 0:
3766 sys
.stdout
.write("Updated branches: ")
3767 for b
in self
.updatedBranches
:
3768 sys
.stdout
.write("%s " % b
)
3769 sys
.stdout
.write("\n")
3771 if gitConfigBool("git-p4.importLabels"):
3772 self
.importLabels
= True
3774 if self
.importLabels
:
3775 p4Labels
= getP4Labels(self
.depotPaths
)
3776 gitTags
= getGitTags()
3778 missingP4Labels
= p4Labels
- gitTags
3779 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3783 # Cleanup temporary branches created during import
3784 if self
.tempBranches
!= []:
3785 for branch
in self
.tempBranches
:
3786 read_pipe("git update-ref -d %s" % branch
)
3787 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3789 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3790 # a convenient shortcut refname "p4".
3791 if self
.importIntoRemotes
:
3792 head_ref
= self
.refPrefix
+ "HEAD"
3793 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3794 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3798 class P4Rebase(Command
):
3800 Command
.__init
__(self
)
3802 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3804 self
.importLabels
= False
3805 self
.description
= ("Fetches the latest revision from perforce and "
3806 + "rebases the current work (branch) against it")
3808 def run(self
, args
):
3810 sync
.importLabels
= self
.importLabels
3813 return self
.rebase()
3816 if os
.system("git update-index --refresh") != 0:
3817 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3818 if len(read_pipe("git diff-index HEAD --")) > 0:
3819 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3821 [upstream
, settings
] = findUpstreamBranchPoint()
3822 if len(upstream
) == 0:
3823 die("Cannot find upstream branchpoint for rebase")
3825 # the branchpoint may be p4/foo~3, so strip off the parent
3826 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3828 print "Rebasing the current branch onto %s" % upstream
3829 oldHead
= read_pipe("git rev-parse HEAD").strip()
3830 system("git rebase %s" % upstream
)
3831 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3834 class P4Clone(P4Sync
):
3836 P4Sync
.__init
__(self
)
3837 self
.description
= "Creates a new git repository and imports from Perforce into it"
3838 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3840 optparse
.make_option("--destination", dest
="cloneDestination",
3841 action
='store', default
=None,
3842 help="where to leave result of the clone"),
3843 optparse
.make_option("--bare", dest
="cloneBare",
3844 action
="store_true", default
=False),
3846 self
.cloneDestination
= None
3847 self
.needsGit
= False
3848 self
.cloneBare
= False
3850 def defaultDestination(self
, args
):
3851 ## TODO: use common prefix of args?
3853 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3854 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3855 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3856 depotDir
= re
.sub(r
"/$", "", depotDir
)
3857 return os
.path
.split(depotDir
)[1]
3859 def run(self
, args
):
3863 if self
.keepRepoPath
and not self
.cloneDestination
:
3864 sys
.stderr
.write("Must specify destination for --keep-path\n")
3869 if not self
.cloneDestination
and len(depotPaths
) > 1:
3870 self
.cloneDestination
= depotPaths
[-1]
3871 depotPaths
= depotPaths
[:-1]
3873 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3874 for p
in depotPaths
:
3875 if not p
.startswith("//"):
3876 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3879 if not self
.cloneDestination
:
3880 self
.cloneDestination
= self
.defaultDestination(args
)
3882 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3884 if not os
.path
.exists(self
.cloneDestination
):
3885 os
.makedirs(self
.cloneDestination
)
3886 chdir(self
.cloneDestination
)
3888 init_cmd
= [ "git", "init" ]
3890 init_cmd
.append("--bare")
3891 retcode
= subprocess
.call(init_cmd
)
3893 raise CalledProcessError(retcode
, init_cmd
)
3895 if not P4Sync
.run(self
, depotPaths
):
3898 # create a master branch and check out a work tree
3899 if gitBranchExists(self
.branch
):
3900 system([ "git", "branch", "master", self
.branch
])
3901 if not self
.cloneBare
:
3902 system([ "git", "checkout", "-f" ])
3904 print 'Not checking out any branch, use ' \
3905 '"git checkout -q -b master <branch>"'
3907 # auto-set this variable if invoked with --use-client-spec
3908 if self
.useClientSpec_from_options
:
3909 system("git config --bool git-p4.useclientspec true")
3913 class P4Unshelve(Command
):
3915 Command
.__init
__(self
)
3917 self
.origin
= "HEAD"
3918 self
.description
= "Unshelve a P4 changelist into a git commit"
3919 self
.usage
= "usage: %prog [options] changelist"
3921 optparse
.make_option("--origin", dest
="origin",
3922 help="Use this base revision instead of the default (%s)" % self
.origin
),
3924 self
.verbose
= False
3925 self
.noCommit
= False
3926 self
.destbranch
= "refs/remotes/p4/unshelved"
3928 def renameBranch(self
, branch_name
):
3929 """ Rename the existing branch to branch_name.N
3933 for i
in range(0,1000):
3934 backup_branch_name
= "{0}.{1}".format(branch_name
, i
)
3935 if not gitBranchExists(backup_branch_name
):
3936 gitUpdateRef(backup_branch_name
, branch_name
) # copy ref to backup
3937 gitDeleteRef(branch_name
)
3939 print("renamed old unshelve branch to {0}".format(backup_branch_name
))
3943 sys
.exit("gave up trying to rename existing branch {0}".format(sync
.branch
))
3945 def findLastP4Revision(self
, starting_point
):
3946 """ Look back from starting_point for the first commit created by git-p4
3947 to find the P4 commit we are based on, and the depot-paths.
3950 for parent
in (range(65535)):
3951 log
= extractLogMessageFromGitCommit("{0}^{1}".format(starting_point
, parent
))
3952 settings
= extractSettingsGitLog(log
)
3953 if 'change' in settings
:
3956 sys
.exit("could not find git-p4 commits in {0}".format(self
.origin
))
3958 def run(self
, args
):
3962 if not gitBranchExists(self
.origin
):
3963 sys
.exit("origin branch {0} does not exist".format(self
.origin
))
3967 sync
.initialParent
= self
.origin
3969 # use the first change in the list to construct the branch to unshelve into
3972 # if the target branch already exists, rename it
3973 branch_name
= "{0}/{1}".format(self
.destbranch
, change
)
3974 if gitBranchExists(branch_name
):
3975 self
.renameBranch(branch_name
)
3976 sync
.branch
= branch_name
3978 sync
.verbose
= self
.verbose
3979 sync
.suppress_meta_comment
= True
3981 settings
= self
.findLastP4Revision(self
.origin
)
3982 origin_revision
= settings
['change']
3983 sync
.depotPaths
= settings
['depot-paths']
3984 sync
.branchPrefixes
= sync
.depotPaths
3987 sync
.loadUserMapFromCache()
3989 sync
.importChanges(changes
, shelved
=True, origin_revision
=origin_revision
)
3992 print("unshelved changelist {0} into {1}".format(change
, branch_name
))
3996 class P4Branches(Command
):
3998 Command
.__init
__(self
)
4000 self
.description
= ("Shows the git branches that hold imports and their "
4001 + "corresponding perforce depot paths")
4002 self
.verbose
= False
4004 def run(self
, args
):
4005 if originP4BranchesExist():
4006 createOrUpdateBranchesFromOrigin()
4008 cmdline
= "git rev-parse --symbolic "
4009 cmdline
+= " --remotes"
4011 for line
in read_pipe_lines(cmdline
):
4014 if not line
.startswith('p4/') or line
== "p4/HEAD":
4018 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
4019 settings
= extractSettingsGitLog(log
)
4021 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
4024 class HelpFormatter(optparse
.IndentedHelpFormatter
):
4026 optparse
.IndentedHelpFormatter
.__init
__(self
)
4028 def format_description(self
, description
):
4030 return description
+ "\n"
4034 def printUsage(commands
):
4035 print "usage: %s <command> [options]" % sys
.argv
[0]
4037 print "valid commands: %s" % ", ".join(commands
)
4039 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
4044 "submit" : P4Submit
,
4045 "commit" : P4Submit
,
4047 "rebase" : P4Rebase
,
4049 "rollback" : P4RollBack
,
4050 "branches" : P4Branches
,
4051 "unshelve" : P4Unshelve
,
4056 if len(sys
.argv
[1:]) == 0:
4057 printUsage(commands
.keys())
4060 cmdName
= sys
.argv
[1]
4062 klass
= commands
[cmdName
]
4065 print "unknown command %s" % cmdName
4067 printUsage(commands
.keys())
4070 options
= cmd
.options
4071 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
4075 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
4077 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
4079 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
4081 description
= cmd
.description
,
4082 formatter
= HelpFormatter())
4084 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
4086 verbose
= cmd
.verbose
4088 if cmd
.gitdir
== None:
4089 cmd
.gitdir
= os
.path
.abspath(".git")
4090 if not isValidGitDir(cmd
.gitdir
):
4091 # "rev-parse --git-dir" without arguments will try $PWD/.git
4092 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
4093 if os
.path
.exists(cmd
.gitdir
):
4094 cdup
= read_pipe("git rev-parse --show-cdup").strip()
4098 if not isValidGitDir(cmd
.gitdir
):
4099 if isValidGitDir(cmd
.gitdir
+ "/.git"):
4100 cmd
.gitdir
+= "/.git"
4102 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
4104 # so git commands invoked from the P4 workspace will succeed
4105 os
.environ
["GIT_DIR"] = cmd
.gitdir
4107 if not cmd
.run(args
):
4112 if __name__
== '__main__':