3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
31 from subprocess
import CalledProcessError
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self
, returncode
, cmd
):
40 self
.returncode
= returncode
43 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize
= 512
53 def p4_build_cmd(cmd
):
54 """Build a suitable p4 command line.
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
62 user
= gitConfig("git-p4.user")
64 real_cmd
+= ["-u",user
]
66 password
= gitConfig("git-p4.password")
68 real_cmd
+= ["-P", password
]
70 port
= gitConfig("git-p4.port")
72 real_cmd
+= ["-p", port
]
74 host
= gitConfig("git-p4.host")
76 real_cmd
+= ["-H", host
]
78 client
= gitConfig("git-p4.client")
80 real_cmd
+= ["-c", client
]
82 retries
= gitConfigInt("git-p4.retries")
84 # Perform 3 retries by default
86 real_cmd
+= ["-r", str(retries
)]
88 if isinstance(cmd
,basestring
):
89 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
95 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
96 This won't automatically add ".git" to a directory.
98 d
= read_pipe(["git", "--git-dir", path
, "rev-parse", "--git-dir"], True).strip()
99 if not d
or len(d
) == 0:
104 def chdir(path
, is_client_path
=False):
105 """Do chdir to the given path, and set the PWD environment
106 variable for use by P4. It does not look at getcwd() output.
107 Since we're not using the shell, it is necessary to set the
108 PWD environment variable explicitly.
110 Normally, expand the path to force it to be absolute. This
111 addresses the use of relative path names inside P4 settings,
112 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
113 as given; it looks for .p4config using PWD.
115 If is_client_path, the path was handed to us directly by p4,
116 and may be a symbolic link. Do not call os.getcwd() in this
117 case, because it will cause p4 to think that PWD is not inside
122 if not is_client_path
:
124 os
.environ
['PWD'] = path
127 """Return free space in bytes on the disk of the given dirname."""
128 if platform
.system() == 'Windows':
129 free_bytes
= ctypes
.c_ulonglong(0)
130 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
131 return free_bytes
.value
133 st
= os
.statvfs(os
.getcwd())
134 return st
.f_bavail
* st
.f_frsize
140 sys
.stderr
.write(msg
+ "\n")
143 def write_pipe(c
, stdin
):
145 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
147 expand
= isinstance(c
,basestring
)
148 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
150 val
= pipe
.write(stdin
)
153 die('Command failed: %s' % str(c
))
157 def p4_write_pipe(c
, stdin
):
158 real_cmd
= p4_build_cmd(c
)
159 return write_pipe(real_cmd
, stdin
)
161 def read_pipe(c
, ignore_error
=False):
163 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
165 expand
= isinstance(c
,basestring
)
166 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
167 (out
, err
) = p
.communicate()
168 if p
.returncode
!= 0 and not ignore_error
:
169 die('Command failed: %s\nError: %s' % (str(c
), err
))
172 def p4_read_pipe(c
, ignore_error
=False):
173 real_cmd
= p4_build_cmd(c
)
174 return read_pipe(real_cmd
, ignore_error
)
176 def read_pipe_lines(c
):
178 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
180 expand
= isinstance(c
, basestring
)
181 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
183 val
= pipe
.readlines()
184 if pipe
.close() or p
.wait():
185 die('Command failed: %s' % str(c
))
189 def p4_read_pipe_lines(c
):
190 """Specifically invoke p4 on the command supplied. """
191 real_cmd
= p4_build_cmd(c
)
192 return read_pipe_lines(real_cmd
)
194 def p4_has_command(cmd
):
195 """Ask p4 for help on this command. If it returns an error, the
196 command does not exist in this version of p4."""
197 real_cmd
= p4_build_cmd(["help", cmd
])
198 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
199 stderr
=subprocess
.PIPE
)
201 return p
.returncode
== 0
203 def p4_has_move_command():
204 """See if the move command exists, that it supports -k, and that
205 it has not been administratively disabled. The arguments
206 must be correct, but the filenames do not have to exist. Use
207 ones with wildcards so even if they exist, it will fail."""
209 if not p4_has_command("move"):
211 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
212 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
213 (out
, err
) = p
.communicate()
214 # return code will be 1 in either case
215 if err
.find("Invalid option") >= 0:
217 if err
.find("disabled") >= 0:
219 # assume it failed because @... was invalid changelist
222 def system(cmd
, ignore_error
=False):
223 expand
= isinstance(cmd
,basestring
)
225 sys
.stderr
.write("executing %s\n" % str(cmd
))
226 retcode
= subprocess
.call(cmd
, shell
=expand
)
227 if retcode
and not ignore_error
:
228 raise CalledProcessError(retcode
, cmd
)
233 """Specifically invoke p4 as the system command. """
234 real_cmd
= p4_build_cmd(cmd
)
235 expand
= isinstance(real_cmd
, basestring
)
236 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
238 raise CalledProcessError(retcode
, real_cmd
)
240 _p4_version_string
= None
241 def p4_version_string():
242 """Read the version string, showing just the last line, which
243 hopefully is the interesting version bit.
246 Perforce - The Fast Software Configuration Management System.
247 Copyright 1995-2011 Perforce Software. All rights reserved.
248 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
250 global _p4_version_string
251 if not _p4_version_string
:
252 a
= p4_read_pipe_lines(["-V"])
253 _p4_version_string
= a
[-1].rstrip()
254 return _p4_version_string
256 def p4_integrate(src
, dest
):
257 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
259 def p4_sync(f
, *options
):
260 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
263 # forcibly add file names with wildcards
264 if wildcard_present(f
):
265 p4_system(["add", "-f", f
])
267 p4_system(["add", f
])
270 p4_system(["delete", wildcard_encode(f
)])
272 def p4_edit(f
, *options
):
273 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
276 p4_system(["revert", wildcard_encode(f
)])
278 def p4_reopen(type, f
):
279 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
281 def p4_reopen_in_change(changelist
, files
):
282 cmd
= ["reopen", "-c", str(changelist
)] + files
285 def p4_move(src
, dest
):
286 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
288 def p4_last_change():
289 results
= p4CmdList(["changes", "-m", "1"])
290 return int(results
[0]['change'])
292 def p4_describe(change
):
293 """Make sure it returns a valid result by checking for
294 the presence of field "time". Return a dict of the
297 ds
= p4CmdList(["describe", "-s", str(change
)])
299 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
303 if "p4ExitCode" in d
:
304 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
307 if d
["code"] == "error":
308 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
311 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
316 # Canonicalize the p4 type and return a tuple of the
317 # base type, plus any modifiers. See "p4 help filetypes"
318 # for a list and explanation.
320 def split_p4_type(p4type
):
322 p4_filetypes_historical
= {
323 "ctempobj": "binary+Sw",
329 "tempobj": "binary+FSw",
330 "ubinary": "binary+F",
331 "uresource": "resource+F",
332 "uxbinary": "binary+Fx",
333 "xbinary": "binary+x",
335 "xtempobj": "binary+Swx",
337 "xunicode": "unicode+x",
340 if p4type
in p4_filetypes_historical
:
341 p4type
= p4_filetypes_historical
[p4type
]
343 s
= p4type
.split("+")
351 # return the raw p4 type of a file (text, text+ko, etc)
354 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
355 return results
[0]['headType']
358 # Given a type base and modifier, return a regexp matching
359 # the keywords that can be expanded in the file
361 def p4_keywords_regexp_for_type(base
, type_mods
):
362 if base
in ("text", "unicode", "binary"):
364 if "ko" in type_mods
:
366 elif "k" in type_mods
:
367 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
371 \$ # Starts with a dollar, followed by...
372 (%s) # one of the keywords, followed by...
373 (:[^$\n]+)? # possibly an old expansion, followed by...
381 # Given a file, return a regexp matching the possible
382 # RCS keywords that will be expanded, or None for files
383 # with kw expansion turned off.
385 def p4_keywords_regexp_for_file(file):
386 if not os
.path
.exists(file):
389 (type_base
, type_mods
) = split_p4_type(p4_type(file))
390 return p4_keywords_regexp_for_type(type_base
, type_mods
)
392 def setP4ExecBit(file, mode
):
393 # Reopens an already open file and changes the execute bit to match
394 # the execute bit setting in the passed in mode.
398 if not isModeExec(mode
):
399 p4Type
= getP4OpenedType(file)
400 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
401 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
402 if p4Type
[-1] == "+":
403 p4Type
= p4Type
[0:-1]
405 p4_reopen(p4Type
, file)
407 def getP4OpenedType(file):
408 # Returns the perforce file type for the given file.
410 result
= p4_read_pipe(["opened", wildcard_encode(file)])
411 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
413 return match
.group(1)
415 die("Could not determine file type for %s (result: '%s')" % (file, result
))
417 # Return the set of all p4 labels
418 def getP4Labels(depotPaths
):
420 if isinstance(depotPaths
,basestring
):
421 depotPaths
= [depotPaths
]
423 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
429 # Return the set of all git tags
432 for line
in read_pipe_lines(["git", "tag"]):
437 def diffTreePattern():
438 # This is a simple generator for the diff tree regex pattern. This could be
439 # a class variable if this and parseDiffTreeEntry were a part of a class.
440 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
444 def parseDiffTreeEntry(entry
):
445 """Parses a single diff tree entry into its component elements.
447 See git-diff-tree(1) manpage for details about the format of the diff
448 output. This method returns a dictionary with the following elements:
450 src_mode - The mode of the source file
451 dst_mode - The mode of the destination file
452 src_sha1 - The sha1 for the source file
453 dst_sha1 - The sha1 fr the destination file
454 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
455 status_score - The score for the status (applicable for 'C' and 'R'
456 statuses). This is None if there is no score.
457 src - The path for the source file.
458 dst - The path for the destination file. This is only present for
459 copy or renames. If it is not present, this is None.
461 If the pattern is not matched, None is returned."""
463 match
= diffTreePattern().next().match(entry
)
466 'src_mode': match
.group(1),
467 'dst_mode': match
.group(2),
468 'src_sha1': match
.group(3),
469 'dst_sha1': match
.group(4),
470 'status': match
.group(5),
471 'status_score': match
.group(6),
472 'src': match
.group(7),
473 'dst': match
.group(10)
477 def isModeExec(mode
):
478 # Returns True if the given git mode represents an executable file,
480 return mode
[-3:] == "755"
482 def isModeExecChanged(src_mode
, dst_mode
):
483 return isModeExec(src_mode
) != isModeExec(dst_mode
)
485 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
487 if isinstance(cmd
,basestring
):
494 cmd
= p4_build_cmd(cmd
)
496 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
498 # Use a temporary file to avoid deadlocks without
499 # subprocess.communicate(), which would put another copy
500 # of stdout into memory.
502 if stdin
is not None:
503 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
504 if isinstance(stdin
,basestring
):
505 stdin_file
.write(stdin
)
508 stdin_file
.write(i
+ '\n')
512 p4
= subprocess
.Popen(cmd
,
515 stdout
=subprocess
.PIPE
)
520 entry
= marshal
.load(p4
.stdout
)
530 entry
["p4ExitCode"] = exitCode
536 list = p4CmdList(cmd
)
542 def p4Where(depotPath
):
543 if not depotPath
.endswith("/"):
545 depotPathLong
= depotPath
+ "..."
546 outputList
= p4CmdList(["where", depotPathLong
])
548 for entry
in outputList
:
549 if "depotFile" in entry
:
550 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
551 # The base path always ends with "/...".
552 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
555 elif "data" in entry
:
556 data
= entry
.get("data")
557 space
= data
.find(" ")
558 if data
[:space
] == depotPath
:
563 if output
["code"] == "error":
567 clientPath
= output
.get("path")
568 elif "data" in output
:
569 data
= output
.get("data")
570 lastSpace
= data
.rfind(" ")
571 clientPath
= data
[lastSpace
+ 1:]
573 if clientPath
.endswith("..."):
574 clientPath
= clientPath
[:-3]
577 def currentGitBranch():
578 retcode
= system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error
=True)
583 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
585 def isValidGitDir(path
):
586 return git_dir(path
) != None
588 def parseRevision(ref
):
589 return read_pipe("git rev-parse %s" % ref
).strip()
591 def branchExists(ref
):
592 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
596 def extractLogMessageFromGitCommit(commit
):
599 ## fixme: title is first line of commit, not 1st paragraph.
601 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
610 def extractSettingsGitLog(log
):
612 for line
in log
.split("\n"):
614 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
618 assignments
= m
.group(1).split (':')
619 for a
in assignments
:
621 key
= vals
[0].strip()
622 val
= ('='.join (vals
[1:])).strip()
623 if val
.endswith ('\"') and val
.startswith('"'):
628 paths
= values
.get("depot-paths")
630 paths
= values
.get("depot-path")
632 values
['depot-paths'] = paths
.split(',')
635 def gitBranchExists(branch
):
636 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
637 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
638 return proc
.wait() == 0;
642 def gitConfig(key
, typeSpecifier
=None):
643 if not _gitConfig
.has_key(key
):
644 cmd
= [ "git", "config" ]
646 cmd
+= [ typeSpecifier
]
648 s
= read_pipe(cmd
, ignore_error
=True)
649 _gitConfig
[key
] = s
.strip()
650 return _gitConfig
[key
]
652 def gitConfigBool(key
):
653 """Return a bool, using git config --bool. It is True only if the
654 variable is set to true, and False if set to false or not present
657 if not _gitConfig
.has_key(key
):
658 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
659 return _gitConfig
[key
]
661 def gitConfigInt(key
):
662 if not _gitConfig
.has_key(key
):
663 cmd
= [ "git", "config", "--int", key
]
664 s
= read_pipe(cmd
, ignore_error
=True)
667 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
669 _gitConfig
[key
] = None
670 return _gitConfig
[key
]
672 def gitConfigList(key
):
673 if not _gitConfig
.has_key(key
):
674 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
675 _gitConfig
[key
] = s
.strip().split(os
.linesep
)
676 if _gitConfig
[key
] == ['']:
678 return _gitConfig
[key
]
680 def p4BranchesInGit(branchesAreInRemotes
=True):
681 """Find all the branches whose names start with "p4/", looking
682 in remotes or heads as specified by the argument. Return
683 a dictionary of { branch: revision } for each one found.
684 The branch names are the short names, without any
689 cmdline
= "git rev-parse --symbolic "
690 if branchesAreInRemotes
:
691 cmdline
+= "--remotes"
693 cmdline
+= "--branches"
695 for line
in read_pipe_lines(cmdline
):
699 if not line
.startswith('p4/'):
701 # special symbolic ref to p4/master
702 if line
== "p4/HEAD":
705 # strip off p4/ prefix
706 branch
= line
[len("p4/"):]
708 branches
[branch
] = parseRevision(line
)
712 def branch_exists(branch
):
713 """Make sure that the given ref name really exists."""
715 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
716 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
717 out
, _
= p
.communicate()
720 # expect exactly one line of output: the branch name
721 return out
.rstrip() == branch
723 def findUpstreamBranchPoint(head
= "HEAD"):
724 branches
= p4BranchesInGit()
725 # map from depot-path to branch name
726 branchByDepotPath
= {}
727 for branch
in branches
.keys():
728 tip
= branches
[branch
]
729 log
= extractLogMessageFromGitCommit(tip
)
730 settings
= extractSettingsGitLog(log
)
731 if settings
.has_key("depot-paths"):
732 paths
= ",".join(settings
["depot-paths"])
733 branchByDepotPath
[paths
] = "remotes/p4/" + branch
737 while parent
< 65535:
738 commit
= head
+ "~%s" % parent
739 log
= extractLogMessageFromGitCommit(commit
)
740 settings
= extractSettingsGitLog(log
)
741 if settings
.has_key("depot-paths"):
742 paths
= ",".join(settings
["depot-paths"])
743 if branchByDepotPath
.has_key(paths
):
744 return [branchByDepotPath
[paths
], settings
]
748 return ["", settings
]
750 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
752 print ("Creating/updating branch(es) in %s based on origin branch(es)"
755 originPrefix
= "origin/p4/"
757 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
759 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
762 headName
= line
[len(originPrefix
):]
763 remoteHead
= localRefPrefix
+ headName
766 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
767 if (not original
.has_key('depot-paths')
768 or not original
.has_key('change')):
772 if not gitBranchExists(remoteHead
):
774 print "creating %s" % remoteHead
777 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
778 if settings
.has_key('change') > 0:
779 if settings
['depot-paths'] == original
['depot-paths']:
780 originP4Change
= int(original
['change'])
781 p4Change
= int(settings
['change'])
782 if originP4Change
> p4Change
:
783 print ("%s (%s) is newer than %s (%s). "
784 "Updating p4 branch from origin."
785 % (originHead
, originP4Change
,
786 remoteHead
, p4Change
))
789 print ("Ignoring: %s was imported from %s while "
790 "%s was imported from %s"
791 % (originHead
, ','.join(original
['depot-paths']),
792 remoteHead
, ','.join(settings
['depot-paths'])))
795 system("git update-ref %s %s" % (remoteHead
, originHead
))
797 def originP4BranchesExist():
798 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
801 def p4ParseNumericChangeRange(parts
):
802 changeStart
= int(parts
[0][1:])
803 if parts
[1] == '#head':
804 changeEnd
= p4_last_change()
806 changeEnd
= int(parts
[1])
808 return (changeStart
, changeEnd
)
810 def chooseBlockSize(blockSize
):
814 return defaultBlockSize
816 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
819 # Parse the change range into start and end. Try to find integer
820 # revision ranges as these can be broken up into blocks to avoid
821 # hitting server-side limits (maxrows, maxscanresults). But if
822 # that doesn't work, fall back to using the raw revision specifier
823 # strings, without using block mode.
825 if changeRange
is None or changeRange
== '':
827 changeEnd
= p4_last_change()
828 block_size
= chooseBlockSize(requestedBlockSize
)
830 parts
= changeRange
.split(',')
831 assert len(parts
) == 2
833 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
834 block_size
= chooseBlockSize(requestedBlockSize
)
836 changeStart
= parts
[0][1:]
838 if requestedBlockSize
:
839 die("cannot use --changes-block-size with non-numeric revisions")
844 # Retrieve changes a block at a time, to prevent running
845 # into a MaxResults/MaxScanRows error from the server.
851 end
= min(changeEnd
, changeStart
+ block_size
)
852 revisionRange
= "%d,%d" % (changeStart
, end
)
854 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
857 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
859 # Insert changes in chronological order
860 for line
in reversed(p4_read_pipe_lines(cmd
)):
861 changes
.add(int(line
.split(" ")[1]))
869 changeStart
= end
+ 1
871 changes
= sorted(changes
)
874 def p4PathStartsWith(path
, prefix
):
875 # This method tries to remedy a potential mixed-case issue:
877 # If UserA adds //depot/DirA/file1
878 # and UserB adds //depot/dira/file2
880 # we may or may not have a problem. If you have core.ignorecase=true,
881 # we treat DirA and dira as the same directory
882 if gitConfigBool("core.ignorecase"):
883 return path
.lower().startswith(prefix
.lower())
884 return path
.startswith(prefix
)
887 """Look at the p4 client spec, create a View() object that contains
888 all the mappings, and return it."""
890 specList
= p4CmdList("client -o")
891 if len(specList
) != 1:
892 die('Output from "client -o" is %d lines, expecting 1' %
895 # dictionary of all client parameters
899 client_name
= entry
["Client"]
901 # just the keys that start with "View"
902 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
905 view
= View(client_name
)
907 # append the lines, in order, to the view
908 for view_num
in range(len(view_keys
)):
909 k
= "View%d" % view_num
910 if k
not in view_keys
:
911 die("Expected view key %s missing" % k
)
912 view
.append(entry
[k
])
917 """Grab the client directory."""
919 output
= p4CmdList("client -o")
921 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
924 if "Root" not in entry
:
925 die('Client has no "Root"')
930 # P4 wildcards are not allowed in filenames. P4 complains
931 # if you simply add them, but you can force it with "-f", in
932 # which case it translates them into %xx encoding internally.
934 def wildcard_decode(path
):
935 # Search for and fix just these four characters. Do % last so
936 # that fixing it does not inadvertently create new %-escapes.
937 # Cannot have * in a filename in windows; untested as to
938 # what p4 would do in such a case.
939 if not platform
.system() == "Windows":
940 path
= path
.replace("%2A", "*")
941 path
= path
.replace("%23", "#") \
942 .replace("%40", "@") \
946 def wildcard_encode(path
):
947 # do % first to avoid double-encoding the %s introduced here
948 path
= path
.replace("%", "%25") \
949 .replace("*", "%2A") \
950 .replace("#", "%23") \
954 def wildcard_present(path
):
955 m
= re
.search("[*#@%]", path
)
958 class LargeFileSystem(object):
959 """Base class for large file system support."""
961 def __init__(self
, writeToGitStream
):
962 self
.largeFiles
= set()
963 self
.writeToGitStream
= writeToGitStream
965 def generatePointer(self
, cloneDestination
, contentFile
):
966 """Return the content of a pointer file that is stored in Git instead of
967 the actual content."""
968 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
970 def pushFile(self
, localLargeFile
):
971 """Push the actual content which is not stored in the Git repository to
973 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
975 def hasLargeFileExtension(self
, relPath
):
978 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
982 def generateTempFile(self
, contents
):
983 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
987 return contentFile
.name
989 def exceedsLargeFileThreshold(self
, relPath
, contents
):
990 if gitConfigInt('git-p4.largeFileThreshold'):
991 contentsSize
= sum(len(d
) for d
in contents
)
992 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
994 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
995 contentsSize
= sum(len(d
) for d
in contents
)
996 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
998 contentTempFile
= self
.generateTempFile(contents
)
999 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1000 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
1001 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
1003 compressedContentsSize
= zf
.infolist()[0].compress_size
1004 os
.remove(contentTempFile
)
1005 os
.remove(compressedContentFile
.name
)
1006 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
1010 def addLargeFile(self
, relPath
):
1011 self
.largeFiles
.add(relPath
)
1013 def removeLargeFile(self
, relPath
):
1014 self
.largeFiles
.remove(relPath
)
1016 def isLargeFile(self
, relPath
):
1017 return relPath
in self
.largeFiles
1019 def processContent(self
, git_mode
, relPath
, contents
):
1020 """Processes the content of git fast import. This method decides if a
1021 file is stored in the large file system and handles all necessary
1023 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1024 contentTempFile
= self
.generateTempFile(contents
)
1025 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1026 if pointer_git_mode
:
1027 git_mode
= pointer_git_mode
1029 # Move temp file to final location in large file system
1030 largeFileDir
= os
.path
.dirname(localLargeFile
)
1031 if not os
.path
.isdir(largeFileDir
):
1032 os
.makedirs(largeFileDir
)
1033 shutil
.move(contentTempFile
, localLargeFile
)
1034 self
.addLargeFile(relPath
)
1035 if gitConfigBool('git-p4.largeFilePush'):
1036 self
.pushFile(localLargeFile
)
1038 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1039 return (git_mode
, contents
)
1041 class MockLFS(LargeFileSystem
):
1042 """Mock large file system for testing."""
1044 def generatePointer(self
, contentFile
):
1045 """The pointer content is the original content prefixed with "pointer-".
1046 The local filename of the large file storage is derived from the file content.
1048 with
open(contentFile
, 'r') as f
:
1051 pointerContents
= 'pointer-' + content
1052 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1053 return (gitMode
, pointerContents
, localLargeFile
)
1055 def pushFile(self
, localLargeFile
):
1056 """The remote filename of the large file storage is the same as the local
1057 one but in a different directory.
1059 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1060 if not os
.path
.exists(remotePath
):
1061 os
.makedirs(remotePath
)
1062 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1064 class GitLFS(LargeFileSystem
):
1065 """Git LFS as backend for the git-p4 large file system.
1066 See https://git-lfs.github.com/ for details."""
1068 def __init__(self
, *args
):
1069 LargeFileSystem
.__init
__(self
, *args
)
1070 self
.baseGitAttributes
= []
1072 def generatePointer(self
, contentFile
):
1073 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1074 mode and content which is stored in the Git repository instead of
1075 the actual content. Return also the new location of the actual
1078 if os
.path
.getsize(contentFile
) == 0:
1079 return (None, '', None)
1081 pointerProcess
= subprocess
.Popen(
1082 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1083 stdout
=subprocess
.PIPE
1085 pointerFile
= pointerProcess
.stdout
.read()
1086 if pointerProcess
.wait():
1087 os
.remove(contentFile
)
1088 die('git-lfs pointer command failed. Did you install the extension?')
1090 # Git LFS removed the preamble in the output of the 'pointer' command
1091 # starting from version 1.2.0. Check for the preamble here to support
1093 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1094 if pointerFile
.startswith('Git LFS pointer for'):
1095 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1097 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1098 localLargeFile
= os
.path
.join(
1100 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1103 # LFS Spec states that pointer files should not have the executable bit set.
1105 return (gitMode
, pointerFile
, localLargeFile
)
1107 def pushFile(self
, localLargeFile
):
1108 uploadProcess
= subprocess
.Popen(
1109 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1111 if uploadProcess
.wait():
1112 die('git-lfs push command failed. Did you define a remote?')
1114 def generateGitAttributes(self
):
1116 self
.baseGitAttributes
+
1120 '# Git LFS (see https://git-lfs.github.com/)\n',
1123 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1124 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1126 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1127 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1131 def addLargeFile(self
, relPath
):
1132 LargeFileSystem
.addLargeFile(self
, relPath
)
1133 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1135 def removeLargeFile(self
, relPath
):
1136 LargeFileSystem
.removeLargeFile(self
, relPath
)
1137 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1139 def processContent(self
, git_mode
, relPath
, contents
):
1140 if relPath
== '.gitattributes':
1141 self
.baseGitAttributes
= contents
1142 return (git_mode
, self
.generateGitAttributes())
1144 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1148 self
.usage
= "usage: %prog [options]"
1149 self
.needsGit
= True
1150 self
.verbose
= False
1154 self
.userMapFromPerforceServer
= False
1155 self
.myP4UserId
= None
1159 return self
.myP4UserId
1161 results
= p4CmdList("user -o")
1163 if r
.has_key('User'):
1164 self
.myP4UserId
= r
['User']
1166 die("Could not find your p4 user id")
1168 def p4UserIsMe(self
, p4User
):
1169 # return True if the given p4 user is actually me
1170 me
= self
.p4UserId()
1171 if not p4User
or p4User
!= me
:
1176 def getUserCacheFilename(self
):
1177 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1178 return home
+ "/.gitp4-usercache.txt"
1180 def getUserMapFromPerforceServer(self
):
1181 if self
.userMapFromPerforceServer
:
1186 for output
in p4CmdList("users"):
1187 if not output
.has_key("User"):
1189 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1190 self
.emails
[output
["Email"]] = output
["User"]
1192 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1193 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1194 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1195 if mapUser
and len(mapUser
[0]) == 3:
1196 user
= mapUser
[0][0]
1197 fullname
= mapUser
[0][1]
1198 email
= mapUser
[0][2]
1199 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1200 self
.emails
[email
] = user
1203 for (key
, val
) in self
.users
.items():
1204 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1206 open(self
.getUserCacheFilename(), "wb").write(s
)
1207 self
.userMapFromPerforceServer
= True
1209 def loadUserMapFromCache(self
):
1211 self
.userMapFromPerforceServer
= False
1213 cache
= open(self
.getUserCacheFilename(), "rb")
1214 lines
= cache
.readlines()
1217 entry
= line
.strip().split("\t")
1218 self
.users
[entry
[0]] = entry
[1]
1220 self
.getUserMapFromPerforceServer()
1222 class P4Debug(Command
):
1224 Command
.__init
__(self
)
1226 self
.description
= "A tool to debug the output of p4 -G."
1227 self
.needsGit
= False
1229 def run(self
, args
):
1231 for output
in p4CmdList(args
):
1232 print 'Element: %d' % j
1237 class P4RollBack(Command
):
1239 Command
.__init
__(self
)
1241 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1243 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1244 self
.rollbackLocalBranches
= False
1246 def run(self
, args
):
1249 maxChange
= int(args
[0])
1251 if "p4ExitCode" in p4Cmd("changes -m 1"):
1252 die("Problems executing p4");
1254 if self
.rollbackLocalBranches
:
1255 refPrefix
= "refs/heads/"
1256 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1258 refPrefix
= "refs/remotes/"
1259 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1262 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1264 ref
= refPrefix
+ line
1265 log
= extractLogMessageFromGitCommit(ref
)
1266 settings
= extractSettingsGitLog(log
)
1268 depotPaths
= settings
['depot-paths']
1269 change
= settings
['change']
1273 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1274 for p
in depotPaths
]))) == 0:
1275 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1276 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1279 while change
and int(change
) > maxChange
:
1282 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1283 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1284 log
= extractLogMessageFromGitCommit(ref
)
1285 settings
= extractSettingsGitLog(log
)
1288 depotPaths
= settings
['depot-paths']
1289 change
= settings
['change']
1292 print "%s rewound to %s" % (ref
, change
)
1296 class P4Submit(Command
, P4UserMap
):
1298 conflict_behavior_choices
= ("ask", "skip", "quit")
1301 Command
.__init
__(self
)
1302 P4UserMap
.__init
__(self
)
1304 optparse
.make_option("--origin", dest
="origin"),
1305 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1306 # preserve the user, requires relevant p4 permissions
1307 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1308 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1309 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1310 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1311 optparse
.make_option("--conflict", dest
="conflict_behavior",
1312 choices
=self
.conflict_behavior_choices
),
1313 optparse
.make_option("--branch", dest
="branch"),
1314 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1315 help="Shelve instead of submit. Shelved files are reverted, "
1316 "restoring the workspace to the state before the shelve"),
1317 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="store", type="int",
1318 metavar
="CHANGELIST",
1319 help="update an existing shelved changelist, implies --shelve")
1321 self
.description
= "Submit changes from git to the perforce depot."
1322 self
.usage
+= " [name of git branch to submit into perforce depot]"
1324 self
.detectRenames
= False
1325 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1326 self
.dry_run
= False
1328 self
.update_shelve
= None
1329 self
.prepare_p4_only
= False
1330 self
.conflict_behavior
= None
1331 self
.isWindows
= (platform
.system() == "Windows")
1332 self
.exportLabels
= False
1333 self
.p4HasMoveCommand
= p4_has_move_command()
1336 if gitConfig('git-p4.largeFileSystem'):
1337 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1340 if len(p4CmdList("opened ...")) > 0:
1341 die("You have files opened with perforce! Close them before starting the sync.")
1343 def separate_jobs_from_description(self
, message
):
1344 """Extract and return a possible Jobs field in the commit
1345 message. It goes into a separate section in the p4 change
1348 A jobs line starts with "Jobs:" and looks like a new field
1349 in a form. Values are white-space separated on the same
1350 line or on following lines that start with a tab.
1352 This does not parse and extract the full git commit message
1353 like a p4 form. It just sees the Jobs: line as a marker
1354 to pass everything from then on directly into the p4 form,
1355 but outside the description section.
1357 Return a tuple (stripped log message, jobs string)."""
1359 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1361 return (message
, None)
1363 jobtext
= message
[m
.start():]
1364 stripped_message
= message
[:m
.start()].rstrip()
1365 return (stripped_message
, jobtext
)
1367 def prepareLogMessage(self
, template
, message
, jobs
):
1368 """Edits the template returned from "p4 change -o" to insert
1369 the message in the Description field, and the jobs text in
1373 inDescriptionSection
= False
1375 for line
in template
.split("\n"):
1376 if line
.startswith("#"):
1377 result
+= line
+ "\n"
1380 if inDescriptionSection
:
1381 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1382 inDescriptionSection
= False
1383 # insert Jobs section
1385 result
+= jobs
+ "\n"
1389 if line
.startswith("Description:"):
1390 inDescriptionSection
= True
1392 for messageLine
in message
.split("\n"):
1393 line
+= "\t" + messageLine
+ "\n"
1395 result
+= line
+ "\n"
1399 def patchRCSKeywords(self
, file, pattern
):
1400 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1401 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1403 outFile
= os
.fdopen(handle
, "w+")
1404 inFile
= open(file, "r")
1405 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1406 for line
in inFile
.readlines():
1407 line
= regexp
.sub(r
'$\1$', line
)
1411 # Forcibly overwrite the original file
1413 shutil
.move(outFileName
, file)
1415 # cleanup our temporary file
1416 os
.unlink(outFileName
)
1417 print "Failed to strip RCS keywords in %s" % file
1420 print "Patched up RCS keywords in %s" % file
1422 def p4UserForCommit(self
,id):
1423 # Return the tuple (perforce user,git email) for a given git commit id
1424 self
.getUserMapFromPerforceServer()
1425 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1426 "--format=%ae", id])
1427 gitEmail
= gitEmail
.strip()
1428 if not self
.emails
.has_key(gitEmail
):
1429 return (None,gitEmail
)
1431 return (self
.emails
[gitEmail
],gitEmail
)
1433 def checkValidP4Users(self
,commits
):
1434 # check if any git authors cannot be mapped to p4 users
1436 (user
,email
) = self
.p4UserForCommit(id)
1438 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1439 if gitConfigBool("git-p4.allowMissingP4Users"):
1442 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1444 def lastP4Changelist(self
):
1445 # Get back the last changelist number submitted in this client spec. This
1446 # then gets used to patch up the username in the change. If the same
1447 # client spec is being used by multiple processes then this might go
1449 results
= p4CmdList("client -o") # find the current client
1452 if r
.has_key('Client'):
1453 client
= r
['Client']
1456 die("could not get client spec")
1457 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1459 if r
.has_key('change'):
1461 die("Could not get changelist number for last submit - cannot patch up user details")
1463 def modifyChangelistUser(self
, changelist
, newUser
):
1464 # fixup the user field of a changelist after it has been submitted.
1465 changes
= p4CmdList("change -o %s" % changelist
)
1466 if len(changes
) != 1:
1467 die("Bad output from p4 change modifying %s to user %s" %
1468 (changelist
, newUser
))
1471 if c
['User'] == newUser
: return # nothing to do
1473 input = marshal
.dumps(c
)
1475 result
= p4CmdList("change -f -i", stdin
=input)
1477 if r
.has_key('code'):
1478 if r
['code'] == 'error':
1479 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1480 if r
.has_key('data'):
1481 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1483 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1485 def canChangeChangelists(self
):
1486 # check to see if we have p4 admin or super-user permissions, either of
1487 # which are required to modify changelists.
1488 results
= p4CmdList(["protects", self
.depotPath
])
1490 if r
.has_key('perm'):
1491 if r
['perm'] == 'admin':
1493 if r
['perm'] == 'super':
1497 def prepareSubmitTemplate(self
, changelist
=None):
1498 """Run "p4 change -o" to grab a change specification template.
1499 This does not use "p4 -G", as it is nice to keep the submission
1500 template in original order, since a human might edit it.
1502 Remove lines in the Files section that show changes to files
1503 outside the depot path we're committing into."""
1505 [upstream
, settings
] = findUpstreamBranchPoint()
1508 inFilesSection
= False
1509 args
= ['change', '-o']
1511 args
.append(str(changelist
))
1513 for line
in p4_read_pipe_lines(args
):
1514 if line
.endswith("\r\n"):
1515 line
= line
[:-2] + "\n"
1517 if line
.startswith("\t"):
1518 # path starts and ends with a tab
1520 lastTab
= path
.rfind("\t")
1522 path
= path
[:lastTab
]
1523 if settings
.has_key('depot-paths'):
1524 if not [p
for p
in settings
['depot-paths']
1525 if p4PathStartsWith(path
, p
)]:
1528 if not p4PathStartsWith(path
, self
.depotPath
):
1531 inFilesSection
= False
1533 if line
.startswith("Files:"):
1534 inFilesSection
= True
1540 def edit_template(self
, template_file
):
1541 """Invoke the editor to let the user change the submission
1542 message. Return true if okay to continue with the submit."""
1544 # if configured to skip the editing part, just submit
1545 if gitConfigBool("git-p4.skipSubmitEdit"):
1548 # look at the modification time, to check later if the user saved
1550 mtime
= os
.stat(template_file
).st_mtime
1553 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1554 editor
= os
.environ
.get("P4EDITOR")
1556 editor
= read_pipe("git var GIT_EDITOR").strip()
1557 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1559 # If the file was not saved, prompt to see if this patch should
1560 # be skipped. But skip this verification step if configured so.
1561 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1564 # modification time updated means user saved the file
1565 if os
.stat(template_file
).st_mtime
> mtime
:
1569 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1575 def get_diff_description(self
, editedFiles
, filesToAdd
, symlinks
):
1577 if os
.environ
.has_key("P4DIFF"):
1578 del(os
.environ
["P4DIFF"])
1580 for editedFile
in editedFiles
:
1581 diff
+= p4_read_pipe(['diff', '-du',
1582 wildcard_encode(editedFile
)])
1586 for newFile
in filesToAdd
:
1587 newdiff
+= "==== new file ====\n"
1588 newdiff
+= "--- /dev/null\n"
1589 newdiff
+= "+++ %s\n" % newFile
1591 is_link
= os
.path
.islink(newFile
)
1592 expect_link
= newFile
in symlinks
1594 if is_link
and expect_link
:
1595 newdiff
+= "+%s\n" % os
.readlink(newFile
)
1597 f
= open(newFile
, "r")
1598 for line
in f
.readlines():
1599 newdiff
+= "+" + line
1602 return (diff
+ newdiff
).replace('\r\n', '\n')
1604 def applyCommit(self
, id):
1605 """Apply one commit, return True if it succeeded."""
1607 print "Applying", read_pipe(["git", "show", "-s",
1608 "--format=format:%h %s", id])
1610 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1612 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1614 filesToChangeType
= set()
1615 filesToDelete
= set()
1617 pureRenameCopy
= set()
1619 filesToChangeExecBit
= {}
1623 diff
= parseDiffTreeEntry(line
)
1624 modifier
= diff
['status']
1626 all_files
.append(path
)
1630 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1631 filesToChangeExecBit
[path
] = diff
['dst_mode']
1632 editedFiles
.add(path
)
1633 elif modifier
== "A":
1634 filesToAdd
.add(path
)
1635 filesToChangeExecBit
[path
] = diff
['dst_mode']
1636 if path
in filesToDelete
:
1637 filesToDelete
.remove(path
)
1639 dst_mode
= int(diff
['dst_mode'], 8)
1640 if dst_mode
== 0120000:
1643 elif modifier
== "D":
1644 filesToDelete
.add(path
)
1645 if path
in filesToAdd
:
1646 filesToAdd
.remove(path
)
1647 elif modifier
== "C":
1648 src
, dest
= diff
['src'], diff
['dst']
1649 p4_integrate(src
, dest
)
1650 pureRenameCopy
.add(dest
)
1651 if diff
['src_sha1'] != diff
['dst_sha1']:
1653 pureRenameCopy
.discard(dest
)
1654 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1656 pureRenameCopy
.discard(dest
)
1657 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1659 # turn off read-only attribute
1660 os
.chmod(dest
, stat
.S_IWRITE
)
1662 editedFiles
.add(dest
)
1663 elif modifier
== "R":
1664 src
, dest
= diff
['src'], diff
['dst']
1665 if self
.p4HasMoveCommand
:
1666 p4_edit(src
) # src must be open before move
1667 p4_move(src
, dest
) # opens for (move/delete, move/add)
1669 p4_integrate(src
, dest
)
1670 if diff
['src_sha1'] != diff
['dst_sha1']:
1673 pureRenameCopy
.add(dest
)
1674 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1675 if not self
.p4HasMoveCommand
:
1676 p4_edit(dest
) # with move: already open, writable
1677 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1678 if not self
.p4HasMoveCommand
:
1680 os
.chmod(dest
, stat
.S_IWRITE
)
1682 filesToDelete
.add(src
)
1683 editedFiles
.add(dest
)
1684 elif modifier
== "T":
1685 filesToChangeType
.add(path
)
1687 die("unknown modifier %s for %s" % (modifier
, path
))
1689 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1690 patchcmd
= diffcmd
+ " | git apply "
1691 tryPatchCmd
= patchcmd
+ "--check -"
1692 applyPatchCmd
= patchcmd
+ "--check --apply -"
1693 patch_succeeded
= True
1695 if os
.system(tryPatchCmd
) != 0:
1696 fixed_rcs_keywords
= False
1697 patch_succeeded
= False
1698 print "Unfortunately applying the change failed!"
1700 # Patch failed, maybe it's just RCS keyword woes. Look through
1701 # the patch to see if that's possible.
1702 if gitConfigBool("git-p4.attemptRCSCleanup"):
1706 for file in editedFiles | filesToDelete
:
1707 # did this file's delta contain RCS keywords?
1708 pattern
= p4_keywords_regexp_for_file(file)
1711 # this file is a possibility...look for RCS keywords.
1712 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1713 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1714 if regexp
.search(line
):
1716 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1717 kwfiles
[file] = pattern
1720 for file in kwfiles
:
1722 print "zapping %s with %s" % (line
,pattern
)
1723 # File is being deleted, so not open in p4. Must
1724 # disable the read-only bit on windows.
1725 if self
.isWindows
and file not in editedFiles
:
1726 os
.chmod(file, stat
.S_IWRITE
)
1727 self
.patchRCSKeywords(file, kwfiles
[file])
1728 fixed_rcs_keywords
= True
1730 if fixed_rcs_keywords
:
1731 print "Retrying the patch with RCS keywords cleaned up"
1732 if os
.system(tryPatchCmd
) == 0:
1733 patch_succeeded
= True
1735 if not patch_succeeded
:
1736 for f
in editedFiles
:
1741 # Apply the patch for real, and do add/delete/+x handling.
1743 system(applyPatchCmd
)
1745 for f
in filesToChangeType
:
1746 p4_edit(f
, "-t", "auto")
1747 for f
in filesToAdd
:
1749 for f
in filesToDelete
:
1753 # Set/clear executable bits
1754 for f
in filesToChangeExecBit
.keys():
1755 mode
= filesToChangeExecBit
[f
]
1756 setP4ExecBit(f
, mode
)
1758 if self
.update_shelve
:
1759 print("all_files = %s" % str(all_files
))
1760 p4_reopen_in_change(self
.update_shelve
, all_files
)
1763 # Build p4 change description, starting with the contents
1764 # of the git commit message.
1766 logMessage
= extractLogMessageFromGitCommit(id)
1767 logMessage
= logMessage
.strip()
1768 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1770 template
= self
.prepareSubmitTemplate(self
.update_shelve
)
1771 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1773 if self
.preserveUser
:
1774 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1776 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1777 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1778 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1779 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1781 separatorLine
= "######## everything below this line is just the diff #######\n"
1782 if not self
.prepare_p4_only
:
1783 submitTemplate
+= separatorLine
1784 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
, symlinks
)
1786 (handle
, fileName
) = tempfile
.mkstemp()
1787 tmpFile
= os
.fdopen(handle
, "w+b")
1789 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1790 tmpFile
.write(submitTemplate
)
1793 if self
.prepare_p4_only
:
1795 # Leave the p4 tree prepared, and the submit template around
1796 # and let the user decide what to do next
1799 print "P4 workspace prepared for submission."
1800 print "To submit or revert, go to client workspace"
1801 print " " + self
.clientPath
1803 print "To submit, use \"p4 submit\" to write a new description,"
1804 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1805 " \"git p4\"." % fileName
1806 print "You can delete the file \"%s\" when finished." % fileName
1808 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1809 print "To preserve change ownership by user %s, you must\n" \
1810 "do \"p4 change -f <change>\" after submitting and\n" \
1811 "edit the User field."
1813 print "After submitting, renamed files must be re-synced."
1814 print "Invoke \"p4 sync -f\" on each of these files:"
1815 for f
in pureRenameCopy
:
1819 print "To revert the changes, use \"p4 revert ...\", and delete"
1820 print "the submit template file \"%s\"" % fileName
1822 print "Since the commit adds new files, they must be deleted:"
1823 for f
in filesToAdd
:
1829 # Let the user edit the change description, then submit it.
1834 if self
.edit_template(fileName
):
1835 # read the edited message and submit
1836 tmpFile
= open(fileName
, "rb")
1837 message
= tmpFile
.read()
1840 message
= message
.replace("\r\n", "\n")
1841 submitTemplate
= message
[:message
.index(separatorLine
)]
1843 if self
.update_shelve
:
1844 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
1846 p4_write_pipe(['shelve', '-i'], submitTemplate
)
1848 p4_write_pipe(['submit', '-i'], submitTemplate
)
1849 # The rename/copy happened by applying a patch that created a
1850 # new file. This leaves it writable, which confuses p4.
1851 for f
in pureRenameCopy
:
1854 if self
.preserveUser
:
1856 # Get last changelist number. Cannot easily get it from
1857 # the submit command output as the output is
1859 changelist
= self
.lastP4Changelist()
1860 self
.modifyChangelistUser(changelist
, p4User
)
1866 if not submitted
or self
.shelve
:
1868 print ("Reverting shelved files.")
1870 print ("Submission cancelled, undoing p4 changes.")
1871 for f
in editedFiles | filesToDelete
:
1873 for f
in filesToAdd
:
1880 # Export git tags as p4 labels. Create a p4 label and then tag
1882 def exportGitTags(self
, gitTags
):
1883 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1884 if len(validLabelRegexp
) == 0:
1885 validLabelRegexp
= defaultLabelRegexp
1886 m
= re
.compile(validLabelRegexp
)
1888 for name
in gitTags
:
1890 if not m
.match(name
):
1892 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1895 # Get the p4 commit this corresponds to
1896 logMessage
= extractLogMessageFromGitCommit(name
)
1897 values
= extractSettingsGitLog(logMessage
)
1899 if not values
.has_key('change'):
1900 # a tag pointing to something not sent to p4; ignore
1902 print "git tag %s does not give a p4 commit" % name
1905 changelist
= values
['change']
1907 # Get the tag details.
1911 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1914 if re
.match(r
'tag\s+', l
):
1916 elif re
.match(r
'\s*$', l
):
1923 body
= ["lightweight tag imported by git p4\n"]
1925 # Create the label - use the same view as the client spec we are using
1926 clientSpec
= getClientSpec()
1928 labelTemplate
= "Label: %s\n" % name
1929 labelTemplate
+= "Description:\n"
1931 labelTemplate
+= "\t" + b
+ "\n"
1932 labelTemplate
+= "View:\n"
1933 for depot_side
in clientSpec
.mappings
:
1934 labelTemplate
+= "\t%s\n" % depot_side
1937 print "Would create p4 label %s for tag" % name
1938 elif self
.prepare_p4_only
:
1939 print "Not creating p4 label %s for tag due to option" \
1940 " --prepare-p4-only" % name
1942 p4_write_pipe(["label", "-i"], labelTemplate
)
1945 p4_system(["tag", "-l", name
] +
1946 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
1949 print "created p4 label for tag %s" % name
1951 def run(self
, args
):
1953 self
.master
= currentGitBranch()
1954 elif len(args
) == 1:
1955 self
.master
= args
[0]
1956 if not branchExists(self
.master
):
1957 die("Branch %s does not exist" % self
.master
)
1962 allowSubmit
= gitConfig("git-p4.allowSubmit")
1963 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1964 die("%s is not in git-p4.allowSubmit" % self
.master
)
1966 [upstream
, settings
] = findUpstreamBranchPoint()
1967 self
.depotPath
= settings
['depot-paths'][0]
1968 if len(self
.origin
) == 0:
1969 self
.origin
= upstream
1971 if self
.update_shelve
:
1974 if self
.preserveUser
:
1975 if not self
.canChangeChangelists():
1976 die("Cannot preserve user names without p4 super-user or admin permissions")
1978 # if not set from the command line, try the config file
1979 if self
.conflict_behavior
is None:
1980 val
= gitConfig("git-p4.conflict")
1982 if val
not in self
.conflict_behavior_choices
:
1983 die("Invalid value '%s' for config git-p4.conflict" % val
)
1986 self
.conflict_behavior
= val
1989 print "Origin branch is " + self
.origin
1991 if len(self
.depotPath
) == 0:
1992 print "Internal error: cannot locate perforce depot path from existing branches"
1995 self
.useClientSpec
= False
1996 if gitConfigBool("git-p4.useclientspec"):
1997 self
.useClientSpec
= True
1998 if self
.useClientSpec
:
1999 self
.clientSpecDirs
= getClientSpec()
2001 # Check for the existence of P4 branches
2002 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
2004 if self
.useClientSpec
and not branchesDetected
:
2005 # all files are relative to the client spec
2006 self
.clientPath
= getClientRoot()
2008 self
.clientPath
= p4Where(self
.depotPath
)
2010 if self
.clientPath
== "":
2011 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
2013 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
2014 self
.oldWorkingDirectory
= os
.getcwd()
2016 # ensure the clientPath exists
2017 new_client_dir
= False
2018 if not os
.path
.exists(self
.clientPath
):
2019 new_client_dir
= True
2020 os
.makedirs(self
.clientPath
)
2022 chdir(self
.clientPath
, is_client_path
=True)
2024 print "Would synchronize p4 checkout in %s" % self
.clientPath
2026 print "Synchronizing p4 checkout..."
2028 # old one was destroyed, and maybe nobody told p4
2029 p4_sync("...", "-f")
2036 commitish
= self
.master
2040 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, commitish
)]):
2041 commits
.append(line
.strip())
2044 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2045 self
.checkAuthorship
= False
2047 self
.checkAuthorship
= True
2049 if self
.preserveUser
:
2050 self
.checkValidP4Users(commits
)
2053 # Build up a set of options to be passed to diff when
2054 # submitting each commit to p4.
2056 if self
.detectRenames
:
2057 # command-line -M arg
2058 self
.diffOpts
= "-M"
2060 # If not explicitly set check the config variable
2061 detectRenames
= gitConfig("git-p4.detectRenames")
2063 if detectRenames
.lower() == "false" or detectRenames
== "":
2065 elif detectRenames
.lower() == "true":
2066 self
.diffOpts
= "-M"
2068 self
.diffOpts
= "-M%s" % detectRenames
2070 # no command-line arg for -C or --find-copies-harder, just
2072 detectCopies
= gitConfig("git-p4.detectCopies")
2073 if detectCopies
.lower() == "false" or detectCopies
== "":
2075 elif detectCopies
.lower() == "true":
2076 self
.diffOpts
+= " -C"
2078 self
.diffOpts
+= " -C%s" % detectCopies
2080 if gitConfigBool("git-p4.detectCopiesHarder"):
2081 self
.diffOpts
+= " --find-copies-harder"
2084 # Apply the commits, one at a time. On failure, ask if should
2085 # continue to try the rest of the patches, or quit.
2090 last
= len(commits
) - 1
2091 for i
, commit
in enumerate(commits
):
2093 print " ", read_pipe(["git", "show", "-s",
2094 "--format=format:%h %s", commit
])
2097 ok
= self
.applyCommit(commit
)
2099 applied
.append(commit
)
2101 if self
.prepare_p4_only
and i
< last
:
2102 print "Processing only the first commit due to option" \
2103 " --prepare-p4-only"
2108 # prompt for what to do, or use the option/variable
2109 if self
.conflict_behavior
== "ask":
2110 print "What do you want to do?"
2111 response
= raw_input("[s]kip this commit but apply"
2112 " the rest, or [q]uit? ")
2115 elif self
.conflict_behavior
== "skip":
2117 elif self
.conflict_behavior
== "quit":
2120 die("Unknown conflict_behavior '%s'" %
2121 self
.conflict_behavior
)
2123 if response
[0] == "s":
2124 print "Skipping this commit, but applying the rest"
2126 if response
[0] == "q":
2133 chdir(self
.oldWorkingDirectory
)
2134 shelved_applied
= "shelved" if self
.shelve
else "applied"
2137 elif self
.prepare_p4_only
:
2139 elif len(commits
) == len(applied
):
2140 print ("All commits {0}!".format(shelved_applied
))
2144 sync
.branch
= self
.branch
2151 if len(applied
) == 0:
2152 print ("No commits {0}.".format(shelved_applied
))
2154 print ("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2160 print star
, read_pipe(["git", "show", "-s",
2161 "--format=format:%h %s", c
])
2162 print "You will have to do 'git p4 sync' and rebase."
2164 if gitConfigBool("git-p4.exportLabels"):
2165 self
.exportLabels
= True
2167 if self
.exportLabels
:
2168 p4Labels
= getP4Labels(self
.depotPath
)
2169 gitTags
= getGitTags()
2171 missingGitTags
= gitTags
- p4Labels
2172 self
.exportGitTags(missingGitTags
)
2174 # exit with error unless everything applied perfectly
2175 if len(commits
) != len(applied
):
2181 """Represent a p4 view ("p4 help views"), and map files in a
2182 repo according to the view."""
2184 def __init__(self
, client_name
):
2186 self
.client_prefix
= "//%s/" % client_name
2187 # cache results of "p4 where" to lookup client file locations
2188 self
.client_spec_path_cache
= {}
2190 def append(self
, view_line
):
2191 """Parse a view line, splitting it into depot and client
2192 sides. Append to self.mappings, preserving order. This
2193 is only needed for tag creation."""
2195 # Split the view line into exactly two words. P4 enforces
2196 # structure on these lines that simplifies this quite a bit.
2198 # Either or both words may be double-quoted.
2199 # Single quotes do not matter.
2200 # Double-quote marks cannot occur inside the words.
2201 # A + or - prefix is also inside the quotes.
2202 # There are no quotes unless they contain a space.
2203 # The line is already white-space stripped.
2204 # The two words are separated by a single space.
2206 if view_line
[0] == '"':
2207 # First word is double quoted. Find its end.
2208 close_quote_index
= view_line
.find('"', 1)
2209 if close_quote_index
<= 0:
2210 die("No first-word closing quote found: %s" % view_line
)
2211 depot_side
= view_line
[1:close_quote_index
]
2212 # skip closing quote and space
2213 rhs_index
= close_quote_index
+ 1 + 1
2215 space_index
= view_line
.find(" ")
2216 if space_index
<= 0:
2217 die("No word-splitting space found: %s" % view_line
)
2218 depot_side
= view_line
[0:space_index
]
2219 rhs_index
= space_index
+ 1
2221 # prefix + means overlay on previous mapping
2222 if depot_side
.startswith("+"):
2223 depot_side
= depot_side
[1:]
2225 # prefix - means exclude this path, leave out of mappings
2227 if depot_side
.startswith("-"):
2229 depot_side
= depot_side
[1:]
2232 self
.mappings
.append(depot_side
)
2234 def convert_client_path(self
, clientFile
):
2235 # chop off //client/ part to make it relative
2236 if not clientFile
.startswith(self
.client_prefix
):
2237 die("No prefix '%s' on clientFile '%s'" %
2238 (self
.client_prefix
, clientFile
))
2239 return clientFile
[len(self
.client_prefix
):]
2241 def update_client_spec_path_cache(self
, files
):
2242 """ Caching file paths by "p4 where" batch query """
2244 # List depot file paths exclude that already cached
2245 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2247 if len(fileArgs
) == 0:
2248 return # All files in cache
2250 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2251 for res
in where_result
:
2252 if "code" in res
and res
["code"] == "error":
2253 # assume error is "... file(s) not in client view"
2255 if "clientFile" not in res
:
2256 die("No clientFile in 'p4 where' output")
2258 # it will list all of them, but only one not unmap-ped
2260 if gitConfigBool("core.ignorecase"):
2261 res
['depotFile'] = res
['depotFile'].lower()
2262 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2264 # not found files or unmap files set to ""
2265 for depotFile
in fileArgs
:
2266 if gitConfigBool("core.ignorecase"):
2267 depotFile
= depotFile
.lower()
2268 if depotFile
not in self
.client_spec_path_cache
:
2269 self
.client_spec_path_cache
[depotFile
] = ""
2271 def map_in_client(self
, depot_path
):
2272 """Return the relative location in the client where this
2273 depot file should live. Returns "" if the file should
2274 not be mapped in the client."""
2276 if gitConfigBool("core.ignorecase"):
2277 depot_path
= depot_path
.lower()
2279 if depot_path
in self
.client_spec_path_cache
:
2280 return self
.client_spec_path_cache
[depot_path
]
2282 die( "Error: %s is not found in client spec path" % depot_path
)
2285 class P4Sync(Command
, P4UserMap
):
2286 delete_actions
= ( "delete", "move/delete", "purge" )
2289 Command
.__init
__(self
)
2290 P4UserMap
.__init
__(self
)
2292 optparse
.make_option("--branch", dest
="branch"),
2293 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2294 optparse
.make_option("--changesfile", dest
="changesFile"),
2295 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2296 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2297 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2298 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2299 help="Import into refs/heads/ , not refs/remotes"),
2300 optparse
.make_option("--max-changes", dest
="maxChanges",
2301 help="Maximum number of changes to import"),
2302 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2303 help="Internal block size to use when iteratively calling p4 changes"),
2304 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2305 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2306 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2307 help="Only sync files that are included in the Perforce Client Spec"),
2308 optparse
.make_option("-/", dest
="cloneExclude",
2309 action
="append", type="string",
2310 help="exclude depot path"),
2312 self
.description
= """Imports from Perforce into a git repository.\n
2314 //depot/my/project/ -- to import the current head
2315 //depot/my/project/@all -- to import everything
2316 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2318 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2320 self
.usage
+= " //depot/path[@revRange]"
2322 self
.createdBranches
= set()
2323 self
.committedChanges
= set()
2325 self
.detectBranches
= False
2326 self
.detectLabels
= False
2327 self
.importLabels
= False
2328 self
.changesFile
= ""
2329 self
.syncWithOrigin
= True
2330 self
.importIntoRemotes
= True
2331 self
.maxChanges
= ""
2332 self
.changes_block_size
= None
2333 self
.keepRepoPath
= False
2334 self
.depotPaths
= None
2335 self
.p4BranchesInGit
= []
2336 self
.cloneExclude
= []
2337 self
.useClientSpec
= False
2338 self
.useClientSpec_from_options
= False
2339 self
.clientSpecDirs
= None
2340 self
.tempBranches
= []
2341 self
.tempBranchLocation
= "refs/git-p4-tmp"
2342 self
.largeFileSystem
= None
2344 if gitConfig('git-p4.largeFileSystem'):
2345 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2346 self
.largeFileSystem
= largeFileSystemConstructor(
2347 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2350 if gitConfig("git-p4.syncFromOrigin") == "false":
2351 self
.syncWithOrigin
= False
2353 # This is required for the "append" cloneExclude action
2354 def ensure_value(self
, attr
, value
):
2355 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2356 setattr(self
, attr
, value
)
2357 return getattr(self
, attr
)
2359 # Force a checkpoint in fast-import and wait for it to finish
2360 def checkpoint(self
):
2361 self
.gitStream
.write("checkpoint\n\n")
2362 self
.gitStream
.write("progress checkpoint\n\n")
2363 out
= self
.gitOutput
.readline()
2365 print "checkpoint finished: " + out
2367 def extractFilesFromCommit(self
, commit
):
2368 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2369 for path
in self
.cloneExclude
]
2372 while commit
.has_key("depotFile%s" % fnum
):
2373 path
= commit
["depotFile%s" % fnum
]
2375 if [p
for p
in self
.cloneExclude
2376 if p4PathStartsWith(path
, p
)]:
2379 found
= [p
for p
in self
.depotPaths
2380 if p4PathStartsWith(path
, p
)]
2387 file["rev"] = commit
["rev%s" % fnum
]
2388 file["action"] = commit
["action%s" % fnum
]
2389 file["type"] = commit
["type%s" % fnum
]
2394 def extractJobsFromCommit(self
, commit
):
2397 while commit
.has_key("job%s" % jnum
):
2398 job
= commit
["job%s" % jnum
]
2403 def stripRepoPath(self
, path
, prefixes
):
2404 """When streaming files, this is called to map a p4 depot path
2405 to where it should go in git. The prefixes are either
2406 self.depotPaths, or self.branchPrefixes in the case of
2407 branch detection."""
2409 if self
.useClientSpec
:
2410 # branch detection moves files up a level (the branch name)
2411 # from what client spec interpretation gives
2412 path
= self
.clientSpecDirs
.map_in_client(path
)
2413 if self
.detectBranches
:
2414 for b
in self
.knownBranches
:
2415 if path
.startswith(b
+ "/"):
2416 path
= path
[len(b
)+1:]
2418 elif self
.keepRepoPath
:
2419 # Preserve everything in relative path name except leading
2420 # //depot/; just look at first prefix as they all should
2421 # be in the same depot.
2422 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2423 if p4PathStartsWith(path
, depot
):
2424 path
= path
[len(depot
):]
2428 if p4PathStartsWith(path
, p
):
2429 path
= path
[len(p
):]
2432 path
= wildcard_decode(path
)
2435 def splitFilesIntoBranches(self
, commit
):
2436 """Look at each depotFile in the commit to figure out to what
2437 branch it belongs."""
2439 if self
.clientSpecDirs
:
2440 files
= self
.extractFilesFromCommit(commit
)
2441 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2445 while commit
.has_key("depotFile%s" % fnum
):
2446 path
= commit
["depotFile%s" % fnum
]
2447 found
= [p
for p
in self
.depotPaths
2448 if p4PathStartsWith(path
, p
)]
2455 file["rev"] = commit
["rev%s" % fnum
]
2456 file["action"] = commit
["action%s" % fnum
]
2457 file["type"] = commit
["type%s" % fnum
]
2460 # start with the full relative path where this file would
2462 if self
.useClientSpec
:
2463 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2465 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2467 for branch
in self
.knownBranches
.keys():
2468 # add a trailing slash so that a commit into qt/4.2foo
2469 # doesn't end up in qt/4.2, e.g.
2470 if relPath
.startswith(branch
+ "/"):
2471 if branch
not in branches
:
2472 branches
[branch
] = []
2473 branches
[branch
].append(file)
2478 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2479 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2480 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2482 self
.gitStream
.write(d
)
2483 self
.gitStream
.write('\n')
2485 # output one file from the P4 stream
2486 # - helper for streamP4Files
2488 def streamOneP4File(self
, file, contents
):
2489 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2491 size
= int(self
.stream_file
['fileSize'])
2492 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2495 (type_base
, type_mods
) = split_p4_type(file["type"])
2498 if "x" in type_mods
:
2500 if type_base
== "symlink":
2502 # p4 print on a symlink sometimes contains "target\n";
2503 # if it does, remove the newline
2504 data
= ''.join(contents
)
2506 # Some version of p4 allowed creating a symlink that pointed
2507 # to nothing. This causes p4 errors when checking out such
2508 # a change, and errors here too. Work around it by ignoring
2509 # the bad symlink; hopefully a future change fixes it.
2510 print "\nIgnoring empty symlink in %s" % file['depotFile']
2512 elif data
[-1] == '\n':
2513 contents
= [data
[:-1]]
2517 if type_base
== "utf16":
2518 # p4 delivers different text in the python output to -G
2519 # than it does when using "print -o", or normal p4 client
2520 # operations. utf16 is converted to ascii or utf8, perhaps.
2521 # But ascii text saved as -t utf16 is completely mangled.
2522 # Invoke print -o to get the real contents.
2524 # On windows, the newlines will always be mangled by print, so put
2525 # them back too. This is not needed to the cygwin windows version,
2526 # just the native "NT" type.
2529 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2530 except Exception as e
:
2531 if 'Translation of file content failed' in str(e
):
2532 type_base
= 'binary'
2536 if p4_version_string().find('/NT') >= 0:
2537 text
= text
.replace('\r\n', '\n')
2540 if type_base
== "apple":
2541 # Apple filetype files will be streamed as a concatenation of
2542 # its appledouble header and the contents. This is useless
2543 # on both macs and non-macs. If using "print -q -o xx", it
2544 # will create "xx" with the data, and "%xx" with the header.
2545 # This is also not very useful.
2547 # Ideally, someday, this script can learn how to generate
2548 # appledouble files directly and import those to git, but
2549 # non-mac machines can never find a use for apple filetype.
2550 print "\nIgnoring apple filetype file %s" % file['depotFile']
2553 # Note that we do not try to de-mangle keywords on utf16 files,
2554 # even though in theory somebody may want that.
2555 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2557 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2558 text
= ''.join(contents
)
2559 text
= regexp
.sub(r
'$\1$', text
)
2563 relPath
.decode('ascii')
2566 if gitConfig('git-p4.pathEncoding'):
2567 encoding
= gitConfig('git-p4.pathEncoding')
2568 relPath
= relPath
.decode(encoding
, 'replace').encode('utf8', 'replace')
2570 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, relPath
)
2572 if self
.largeFileSystem
:
2573 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2575 self
.writeToGitStream(git_mode
, relPath
, contents
)
2577 def streamOneP4Deletion(self
, file):
2578 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2580 sys
.stdout
.write("delete %s\n" % relPath
)
2582 self
.gitStream
.write("D %s\n" % relPath
)
2584 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2585 self
.largeFileSystem
.removeLargeFile(relPath
)
2587 # handle another chunk of streaming data
2588 def streamP4FilesCb(self
, marshalled
):
2590 # catch p4 errors and complain
2592 if "code" in marshalled
:
2593 if marshalled
["code"] == "error":
2594 if "data" in marshalled
:
2595 err
= marshalled
["data"].rstrip()
2597 if not err
and 'fileSize' in self
.stream_file
:
2598 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2599 if required_bytes
> 0:
2600 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2601 os
.getcwd(), required_bytes
/1024/1024
2606 if self
.stream_have_file_info
:
2607 if "depotFile" in self
.stream_file
:
2608 f
= self
.stream_file
["depotFile"]
2609 # force a failure in fast-import, else an empty
2610 # commit will be made
2611 self
.gitStream
.write("\n")
2612 self
.gitStream
.write("die-now\n")
2613 self
.gitStream
.close()
2614 # ignore errors, but make sure it exits first
2615 self
.importProcess
.wait()
2617 die("Error from p4 print for %s: %s" % (f
, err
))
2619 die("Error from p4 print: %s" % err
)
2621 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2622 # start of a new file - output the old one first
2623 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2624 self
.stream_file
= {}
2625 self
.stream_contents
= []
2626 self
.stream_have_file_info
= False
2628 # pick up the new file information... for the
2629 # 'data' field we need to append to our array
2630 for k
in marshalled
.keys():
2632 if 'streamContentSize' not in self
.stream_file
:
2633 self
.stream_file
['streamContentSize'] = 0
2634 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2635 self
.stream_contents
.append(marshalled
['data'])
2637 self
.stream_file
[k
] = marshalled
[k
]
2640 'streamContentSize' in self
.stream_file
and
2641 'fileSize' in self
.stream_file
and
2642 'depotFile' in self
.stream_file
):
2643 size
= int(self
.stream_file
["fileSize"])
2645 progress
= 100*self
.stream_file
['streamContentSize']/size
2646 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2649 self
.stream_have_file_info
= True
2651 # Stream directly from "p4 files" into "git fast-import"
2652 def streamP4Files(self
, files
):
2658 filesForCommit
.append(f
)
2659 if f
['action'] in self
.delete_actions
:
2660 filesToDelete
.append(f
)
2662 filesToRead
.append(f
)
2665 for f
in filesToDelete
:
2666 self
.streamOneP4Deletion(f
)
2668 if len(filesToRead
) > 0:
2669 self
.stream_file
= {}
2670 self
.stream_contents
= []
2671 self
.stream_have_file_info
= False
2673 # curry self argument
2674 def streamP4FilesCbSelf(entry
):
2675 self
.streamP4FilesCb(entry
)
2677 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2679 p4CmdList(["-x", "-", "print"],
2681 cb
=streamP4FilesCbSelf
)
2684 if self
.stream_file
.has_key('depotFile'):
2685 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2687 def make_email(self
, userid
):
2688 if userid
in self
.users
:
2689 return self
.users
[userid
]
2691 return "%s <a@b>" % userid
2693 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2694 """ Stream a p4 tag.
2695 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2699 print "writing tag %s for commit %s" % (labelName
, commit
)
2700 gitStream
.write("tag %s\n" % labelName
)
2701 gitStream
.write("from %s\n" % commit
)
2703 if labelDetails
.has_key('Owner'):
2704 owner
= labelDetails
["Owner"]
2708 # Try to use the owner of the p4 label, or failing that,
2709 # the current p4 user id.
2711 email
= self
.make_email(owner
)
2713 email
= self
.make_email(self
.p4UserId())
2714 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2716 gitStream
.write("tagger %s\n" % tagger
)
2718 print "labelDetails=",labelDetails
2719 if labelDetails
.has_key('Description'):
2720 description
= labelDetails
['Description']
2722 description
= 'Label from git p4'
2724 gitStream
.write("data %d\n" % len(description
))
2725 gitStream
.write(description
)
2726 gitStream
.write("\n")
2728 def inClientSpec(self
, path
):
2729 if not self
.clientSpecDirs
:
2731 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
2732 if not inClientSpec
and self
.verbose
:
2733 print('Ignoring file outside of client spec: {0}'.format(path
))
2736 def hasBranchPrefix(self
, path
):
2737 if not self
.branchPrefixes
:
2739 hasPrefix
= [p
for p
in self
.branchPrefixes
2740 if p4PathStartsWith(path
, p
)]
2741 if not hasPrefix
and self
.verbose
:
2742 print('Ignoring file outside of prefix: {0}'.format(path
))
2745 def commit(self
, details
, files
, branch
, parent
= ""):
2746 epoch
= details
["time"]
2747 author
= details
["user"]
2748 jobs
= self
.extractJobsFromCommit(details
)
2751 print('commit into {0}'.format(branch
))
2753 if self
.clientSpecDirs
:
2754 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2756 files
= [f
for f
in files
2757 if self
.inClientSpec(f
['path']) and self
.hasBranchPrefix(f
['path'])]
2759 if not files
and not gitConfigBool('git-p4.keepEmptyCommits'):
2760 print('Ignoring revision {0} as it would produce an empty commit.'
2761 .format(details
['change']))
2764 self
.gitStream
.write("commit %s\n" % branch
)
2765 self
.gitStream
.write("mark :%s\n" % details
["change"])
2766 self
.committedChanges
.add(int(details
["change"]))
2768 if author
not in self
.users
:
2769 self
.getUserMapFromPerforceServer()
2770 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2772 self
.gitStream
.write("committer %s\n" % committer
)
2774 self
.gitStream
.write("data <<EOT\n")
2775 self
.gitStream
.write(details
["desc"])
2777 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
2778 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2779 (','.join(self
.branchPrefixes
), details
["change"]))
2780 if len(details
['options']) > 0:
2781 self
.gitStream
.write(": options = %s" % details
['options'])
2782 self
.gitStream
.write("]\nEOT\n\n")
2786 print "parent %s" % parent
2787 self
.gitStream
.write("from %s\n" % parent
)
2789 self
.streamP4Files(files
)
2790 self
.gitStream
.write("\n")
2792 change
= int(details
["change"])
2794 if self
.labels
.has_key(change
):
2795 label
= self
.labels
[change
]
2796 labelDetails
= label
[0]
2797 labelRevisions
= label
[1]
2799 print "Change %s is labelled %s" % (change
, labelDetails
)
2801 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2802 for p
in self
.branchPrefixes
])
2804 if len(files
) == len(labelRevisions
):
2808 if info
["action"] in self
.delete_actions
:
2810 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2812 if cleanedFiles
== labelRevisions
:
2813 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2817 print ("Tag %s does not match with change %s: files do not match."
2818 % (labelDetails
["label"], change
))
2822 print ("Tag %s does not match with change %s: file count is different."
2823 % (labelDetails
["label"], change
))
2825 # Build a dictionary of changelists and labels, for "detect-labels" option.
2826 def getLabels(self
):
2829 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2830 if len(l
) > 0 and not self
.silent
:
2831 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2834 label
= output
["label"]
2838 print "Querying files for label %s" % label
2839 for file in p4CmdList(["files"] +
2840 ["%s...@%s" % (p
, label
)
2841 for p
in self
.depotPaths
]):
2842 revisions
[file["depotFile"]] = file["rev"]
2843 change
= int(file["change"])
2844 if change
> newestChange
:
2845 newestChange
= change
2847 self
.labels
[newestChange
] = [output
, revisions
]
2850 print "Label changes: %s" % self
.labels
.keys()
2852 # Import p4 labels as git tags. A direct mapping does not
2853 # exist, so assume that if all the files are at the same revision
2854 # then we can use that, or it's something more complicated we should
2856 def importP4Labels(self
, stream
, p4Labels
):
2858 print "import p4 labels: " + ' '.join(p4Labels
)
2860 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2861 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2862 if len(validLabelRegexp
) == 0:
2863 validLabelRegexp
= defaultLabelRegexp
2864 m
= re
.compile(validLabelRegexp
)
2866 for name
in p4Labels
:
2869 if not m
.match(name
):
2871 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2874 if name
in ignoredP4Labels
:
2877 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2879 # get the most recent changelist for each file in this label
2880 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2881 for p
in self
.depotPaths
])
2883 if change
.has_key('change'):
2884 # find the corresponding git commit; take the oldest commit
2885 changelist
= int(change
['change'])
2886 if changelist
in self
.committedChanges
:
2887 gitCommit
= ":%d" % changelist
# use a fast-import mark
2890 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2891 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
2892 if len(gitCommit
) == 0:
2893 print "importing label %s: could not find git commit for changelist %d" % (name
, changelist
)
2896 gitCommit
= gitCommit
.strip()
2899 # Convert from p4 time format
2901 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2903 print "Could not convert label time %s" % labelDetails
['Update']
2906 when
= int(time
.mktime(tmwhen
))
2907 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2909 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2912 print "Label %s has no changelists - possibly deleted?" % name
2915 # We can't import this label; don't try again as it will get very
2916 # expensive repeatedly fetching all the files for labels that will
2917 # never be imported. If the label is moved in the future, the
2918 # ignore will need to be removed manually.
2919 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2921 def guessProjectName(self
):
2922 for p
in self
.depotPaths
:
2925 p
= p
[p
.strip().rfind("/") + 1:]
2926 if not p
.endswith("/"):
2930 def getBranchMapping(self
):
2931 lostAndFoundBranches
= set()
2933 user
= gitConfig("git-p4.branchUser")
2935 command
= "branches -u %s" % user
2937 command
= "branches"
2939 for info
in p4CmdList(command
):
2940 details
= p4Cmd(["branch", "-o", info
["branch"]])
2942 while details
.has_key("View%s" % viewIdx
):
2943 paths
= details
["View%s" % viewIdx
].split(" ")
2944 viewIdx
= viewIdx
+ 1
2945 # require standard //depot/foo/... //depot/bar/... mapping
2946 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2949 destination
= paths
[1]
2951 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2952 source
= source
[len(self
.depotPaths
[0]):-4]
2953 destination
= destination
[len(self
.depotPaths
[0]):-4]
2955 if destination
in self
.knownBranches
:
2957 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2958 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2961 self
.knownBranches
[destination
] = source
2963 lostAndFoundBranches
.discard(destination
)
2965 if source
not in self
.knownBranches
:
2966 lostAndFoundBranches
.add(source
)
2968 # Perforce does not strictly require branches to be defined, so we also
2969 # check git config for a branch list.
2971 # Example of branch definition in git config file:
2973 # branchList=main:branchA
2974 # branchList=main:branchB
2975 # branchList=branchA:branchC
2976 configBranches
= gitConfigList("git-p4.branchList")
2977 for branch
in configBranches
:
2979 (source
, destination
) = branch
.split(":")
2980 self
.knownBranches
[destination
] = source
2982 lostAndFoundBranches
.discard(destination
)
2984 if source
not in self
.knownBranches
:
2985 lostAndFoundBranches
.add(source
)
2988 for branch
in lostAndFoundBranches
:
2989 self
.knownBranches
[branch
] = branch
2991 def getBranchMappingFromGitBranches(self
):
2992 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2993 for branch
in branches
.keys():
2994 if branch
== "master":
2997 branch
= branch
[len(self
.projectName
):]
2998 self
.knownBranches
[branch
] = branch
3000 def updateOptionDict(self
, d
):
3002 if self
.keepRepoPath
:
3003 option_keys
['keepRepoPath'] = 1
3005 d
["options"] = ' '.join(sorted(option_keys
.keys()))
3007 def readOptions(self
, d
):
3008 self
.keepRepoPath
= (d
.has_key('options')
3009 and ('keepRepoPath' in d
['options']))
3011 def gitRefForBranch(self
, branch
):
3012 if branch
== "main":
3013 return self
.refPrefix
+ "master"
3015 if len(branch
) <= 0:
3018 return self
.refPrefix
+ self
.projectName
+ branch
3020 def gitCommitByP4Change(self
, ref
, change
):
3022 print "looking in ref " + ref
+ " for change %s using bisect..." % change
3025 latestCommit
= parseRevision(ref
)
3029 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
3030 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
3035 log
= extractLogMessageFromGitCommit(next
)
3036 settings
= extractSettingsGitLog(log
)
3037 currentChange
= int(settings
['change'])
3039 print "current change %s" % currentChange
3041 if currentChange
== change
:
3043 print "found %s" % next
3046 if currentChange
< change
:
3047 earliestCommit
= "^%s" % next
3049 latestCommit
= "%s" % next
3053 def importNewBranch(self
, branch
, maxChange
):
3054 # make fast-import flush all changes to disk and update the refs using the checkpoint
3055 # command so that we can try to find the branch parent in the git history
3056 self
.gitStream
.write("checkpoint\n\n");
3057 self
.gitStream
.flush();
3058 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3059 range = "@1,%s" % maxChange
3060 #print "prefix" + branchPrefix
3061 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3062 if len(changes
) <= 0:
3064 firstChange
= changes
[0]
3065 #print "first change in branch: %s" % firstChange
3066 sourceBranch
= self
.knownBranches
[branch
]
3067 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3068 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3069 #print "source " + sourceBranch
3071 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3072 #print "branch parent: %s" % branchParentChange
3073 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3074 if len(gitParent
) > 0:
3075 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3076 #print "parent git commit: %s" % gitParent
3078 self
.importChanges(changes
)
3081 def searchParent(self
, parent
, branch
, target
):
3083 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
3084 "--no-merges", parent
]):
3086 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
3089 print "Found parent of %s in commit %s" % (branch
, blob
)
3096 def importChanges(self
, changes
):
3098 for change
in changes
:
3099 description
= p4_describe(change
)
3100 self
.updateOptionDict(description
)
3103 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3108 if self
.detectBranches
:
3109 branches
= self
.splitFilesIntoBranches(description
)
3110 for branch
in branches
.keys():
3112 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3113 self
.branchPrefixes
= [ branchPrefix
]
3117 filesForCommit
= branches
[branch
]
3120 print "branch is %s" % branch
3122 self
.updatedBranches
.add(branch
)
3124 if branch
not in self
.createdBranches
:
3125 self
.createdBranches
.add(branch
)
3126 parent
= self
.knownBranches
[branch
]
3127 if parent
== branch
:
3130 fullBranch
= self
.projectName
+ branch
3131 if fullBranch
not in self
.p4BranchesInGit
:
3133 print("\n Importing new branch %s" % fullBranch
);
3134 if self
.importNewBranch(branch
, change
- 1):
3136 self
.p4BranchesInGit
.append(fullBranch
)
3138 print("\n Resuming with change %s" % change
);
3141 print "parent determined through known branches: %s" % parent
3143 branch
= self
.gitRefForBranch(branch
)
3144 parent
= self
.gitRefForBranch(parent
)
3147 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
3149 if len(parent
) == 0 and branch
in self
.initialParents
:
3150 parent
= self
.initialParents
[branch
]
3151 del self
.initialParents
[branch
]
3155 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3157 print "Creating temporary branch: " + tempBranch
3158 self
.commit(description
, filesForCommit
, tempBranch
)
3159 self
.tempBranches
.append(tempBranch
)
3161 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3163 self
.commit(description
, filesForCommit
, branch
, blob
)
3166 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3167 self
.commit(description
, filesForCommit
, branch
, parent
)
3169 files
= self
.extractFilesFromCommit(description
)
3170 self
.commit(description
, files
, self
.branch
,
3172 # only needed once, to connect to the previous commit
3173 self
.initialParent
= ""
3175 print self
.gitError
.read()
3178 def importHeadRevision(self
, revision
):
3179 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3182 details
["user"] = "git perforce import user"
3183 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3184 % (' '.join(self
.depotPaths
), revision
))
3185 details
["change"] = revision
3189 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3191 for info
in p4CmdList(["files"] + fileArgs
):
3193 if 'code' in info
and info
['code'] == 'error':
3194 sys
.stderr
.write("p4 returned an error: %s\n"
3196 if info
['data'].find("must refer to client") >= 0:
3197 sys
.stderr
.write("This particular p4 error is misleading.\n")
3198 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3199 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3201 if 'p4ExitCode' in info
:
3202 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3206 change
= int(info
["change"])
3207 if change
> newestRevision
:
3208 newestRevision
= change
3210 if info
["action"] in self
.delete_actions
:
3211 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3212 #fileCnt = fileCnt + 1
3215 for prop
in ["depotFile", "rev", "action", "type" ]:
3216 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3218 fileCnt
= fileCnt
+ 1
3220 details
["change"] = newestRevision
3222 # Use time from top-most change so that all git p4 clones of
3223 # the same p4 repo have the same commit SHA1s.
3224 res
= p4_describe(newestRevision
)
3225 details
["time"] = res
["time"]
3227 self
.updateOptionDict(details
)
3229 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3231 print "IO error with git fast-import. Is your git version recent enough?"
3232 print self
.gitError
.read()
3235 def run(self
, args
):
3236 self
.depotPaths
= []
3237 self
.changeRange
= ""
3238 self
.previousDepotPaths
= []
3239 self
.hasOrigin
= False
3241 # map from branch depot path to parent branch
3242 self
.knownBranches
= {}
3243 self
.initialParents
= {}
3245 if self
.importIntoRemotes
:
3246 self
.refPrefix
= "refs/remotes/p4/"
3248 self
.refPrefix
= "refs/heads/p4/"
3250 if self
.syncWithOrigin
:
3251 self
.hasOrigin
= originP4BranchesExist()
3254 print 'Syncing with origin first, using "git fetch origin"'
3255 system("git fetch origin")
3257 branch_arg_given
= bool(self
.branch
)
3258 if len(self
.branch
) == 0:
3259 self
.branch
= self
.refPrefix
+ "master"
3260 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3261 system("git update-ref %s refs/heads/p4" % self
.branch
)
3262 system("git branch -D p4")
3264 # accept either the command-line option, or the configuration variable
3265 if self
.useClientSpec
:
3266 # will use this after clone to set the variable
3267 self
.useClientSpec_from_options
= True
3269 if gitConfigBool("git-p4.useclientspec"):
3270 self
.useClientSpec
= True
3271 if self
.useClientSpec
:
3272 self
.clientSpecDirs
= getClientSpec()
3274 # TODO: should always look at previous commits,
3275 # merge with previous imports, if possible.
3278 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3280 # branches holds mapping from branch name to sha1
3281 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3283 # restrict to just this one, disabling detect-branches
3284 if branch_arg_given
:
3285 short
= self
.branch
.split("/")[-1]
3286 if short
in branches
:
3287 self
.p4BranchesInGit
= [ short
]
3289 self
.p4BranchesInGit
= branches
.keys()
3291 if len(self
.p4BranchesInGit
) > 1:
3293 print "Importing from/into multiple branches"
3294 self
.detectBranches
= True
3295 for branch
in branches
.keys():
3296 self
.initialParents
[self
.refPrefix
+ branch
] = \
3300 print "branches: %s" % self
.p4BranchesInGit
3303 for branch
in self
.p4BranchesInGit
:
3304 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3306 settings
= extractSettingsGitLog(logMsg
)
3308 self
.readOptions(settings
)
3309 if (settings
.has_key('depot-paths')
3310 and settings
.has_key ('change')):
3311 change
= int(settings
['change']) + 1
3312 p4Change
= max(p4Change
, change
)
3314 depotPaths
= sorted(settings
['depot-paths'])
3315 if self
.previousDepotPaths
== []:
3316 self
.previousDepotPaths
= depotPaths
3319 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3320 prev_list
= prev
.split("/")
3321 cur_list
= cur
.split("/")
3322 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3323 if cur_list
[i
] <> prev_list
[i
]:
3327 paths
.append ("/".join(cur_list
[:i
+ 1]))
3329 self
.previousDepotPaths
= paths
3332 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3333 self
.changeRange
= "@%s,#head" % p4Change
3334 if not self
.silent
and not self
.detectBranches
:
3335 print "Performing incremental import into %s git branch" % self
.branch
3337 # accept multiple ref name abbreviations:
3338 # refs/foo/bar/branch -> use it exactly
3339 # p4/branch -> prepend refs/remotes/ or refs/heads/
3340 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3341 if not self
.branch
.startswith("refs/"):
3342 if self
.importIntoRemotes
:
3343 prepend
= "refs/remotes/"
3345 prepend
= "refs/heads/"
3346 if not self
.branch
.startswith("p4/"):
3348 self
.branch
= prepend
+ self
.branch
3350 if len(args
) == 0 and self
.depotPaths
:
3352 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3354 if self
.depotPaths
and self
.depotPaths
!= args
:
3355 print ("previous import used depot path %s and now %s was specified. "
3356 "This doesn't work!" % (' '.join (self
.depotPaths
),
3360 self
.depotPaths
= sorted(args
)
3365 # Make sure no revision specifiers are used when --changesfile
3367 bad_changesfile
= False
3368 if len(self
.changesFile
) > 0:
3369 for p
in self
.depotPaths
:
3370 if p
.find("@") >= 0 or p
.find("#") >= 0:
3371 bad_changesfile
= True
3374 die("Option --changesfile is incompatible with revision specifiers")
3377 for p
in self
.depotPaths
:
3378 if p
.find("@") != -1:
3379 atIdx
= p
.index("@")
3380 self
.changeRange
= p
[atIdx
:]
3381 if self
.changeRange
== "@all":
3382 self
.changeRange
= ""
3383 elif ',' not in self
.changeRange
:
3384 revision
= self
.changeRange
3385 self
.changeRange
= ""
3387 elif p
.find("#") != -1:
3388 hashIdx
= p
.index("#")
3389 revision
= p
[hashIdx
:]
3391 elif self
.previousDepotPaths
== []:
3392 # pay attention to changesfile, if given, else import
3393 # the entire p4 tree at the head revision
3394 if len(self
.changesFile
) == 0:
3397 p
= re
.sub ("\.\.\.$", "", p
)
3398 if not p
.endswith("/"):
3403 self
.depotPaths
= newPaths
3405 # --detect-branches may change this for each branch
3406 self
.branchPrefixes
= self
.depotPaths
3408 self
.loadUserMapFromCache()
3410 if self
.detectLabels
:
3413 if self
.detectBranches
:
3414 ## FIXME - what's a P4 projectName ?
3415 self
.projectName
= self
.guessProjectName()
3418 self
.getBranchMappingFromGitBranches()
3420 self
.getBranchMapping()
3422 print "p4-git branches: %s" % self
.p4BranchesInGit
3423 print "initial parents: %s" % self
.initialParents
3424 for b
in self
.p4BranchesInGit
:
3428 b
= b
[len(self
.projectName
):]
3429 self
.createdBranches
.add(b
)
3431 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
3433 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3434 stdin
=subprocess
.PIPE
,
3435 stdout
=subprocess
.PIPE
,
3436 stderr
=subprocess
.PIPE
);
3437 self
.gitOutput
= self
.importProcess
.stdout
3438 self
.gitStream
= self
.importProcess
.stdin
3439 self
.gitError
= self
.importProcess
.stderr
3442 self
.importHeadRevision(revision
)
3446 if len(self
.changesFile
) > 0:
3447 output
= open(self
.changesFile
).readlines()
3450 changeSet
.add(int(line
))
3452 for change
in changeSet
:
3453 changes
.append(change
)
3457 # catch "git p4 sync" with no new branches, in a repo that
3458 # does not have any existing p4 branches
3460 if not self
.p4BranchesInGit
:
3461 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3463 # The default branch is master, unless --branch is used to
3464 # specify something else. Make sure it exists, or complain
3465 # nicely about how to use --branch.
3466 if not self
.detectBranches
:
3467 if not branch_exists(self
.branch
):
3468 if branch_arg_given
:
3469 die("Error: branch %s does not exist." % self
.branch
)
3471 die("Error: no branch %s; perhaps specify one with --branch." %
3475 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3477 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3479 if len(self
.maxChanges
) > 0:
3480 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3482 if len(changes
) == 0:
3484 print "No changes to import!"
3486 if not self
.silent
and not self
.detectBranches
:
3487 print "Import destination: %s" % self
.branch
3489 self
.updatedBranches
= set()
3491 if not self
.detectBranches
:
3493 # start a new branch
3494 self
.initialParent
= ""
3496 # build on a previous revision
3497 self
.initialParent
= parseRevision(self
.branch
)
3499 self
.importChanges(changes
)
3503 if len(self
.updatedBranches
) > 0:
3504 sys
.stdout
.write("Updated branches: ")
3505 for b
in self
.updatedBranches
:
3506 sys
.stdout
.write("%s " % b
)
3507 sys
.stdout
.write("\n")
3509 if gitConfigBool("git-p4.importLabels"):
3510 self
.importLabels
= True
3512 if self
.importLabels
:
3513 p4Labels
= getP4Labels(self
.depotPaths
)
3514 gitTags
= getGitTags()
3516 missingP4Labels
= p4Labels
- gitTags
3517 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3519 self
.gitStream
.close()
3520 if self
.importProcess
.wait() != 0:
3521 die("fast-import failed: %s" % self
.gitError
.read())
3522 self
.gitOutput
.close()
3523 self
.gitError
.close()
3525 # Cleanup temporary branches created during import
3526 if self
.tempBranches
!= []:
3527 for branch
in self
.tempBranches
:
3528 read_pipe("git update-ref -d %s" % branch
)
3529 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3531 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3532 # a convenient shortcut refname "p4".
3533 if self
.importIntoRemotes
:
3534 head_ref
= self
.refPrefix
+ "HEAD"
3535 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3536 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3540 class P4Rebase(Command
):
3542 Command
.__init
__(self
)
3544 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3546 self
.importLabels
= False
3547 self
.description
= ("Fetches the latest revision from perforce and "
3548 + "rebases the current work (branch) against it")
3550 def run(self
, args
):
3552 sync
.importLabels
= self
.importLabels
3555 return self
.rebase()
3558 if os
.system("git update-index --refresh") != 0:
3559 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3560 if len(read_pipe("git diff-index HEAD --")) > 0:
3561 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3563 [upstream
, settings
] = findUpstreamBranchPoint()
3564 if len(upstream
) == 0:
3565 die("Cannot find upstream branchpoint for rebase")
3567 # the branchpoint may be p4/foo~3, so strip off the parent
3568 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3570 print "Rebasing the current branch onto %s" % upstream
3571 oldHead
= read_pipe("git rev-parse HEAD").strip()
3572 system("git rebase %s" % upstream
)
3573 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3576 class P4Clone(P4Sync
):
3578 P4Sync
.__init
__(self
)
3579 self
.description
= "Creates a new git repository and imports from Perforce into it"
3580 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3582 optparse
.make_option("--destination", dest
="cloneDestination",
3583 action
='store', default
=None,
3584 help="where to leave result of the clone"),
3585 optparse
.make_option("--bare", dest
="cloneBare",
3586 action
="store_true", default
=False),
3588 self
.cloneDestination
= None
3589 self
.needsGit
= False
3590 self
.cloneBare
= False
3592 def defaultDestination(self
, args
):
3593 ## TODO: use common prefix of args?
3595 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3596 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3597 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3598 depotDir
= re
.sub(r
"/$", "", depotDir
)
3599 return os
.path
.split(depotDir
)[1]
3601 def run(self
, args
):
3605 if self
.keepRepoPath
and not self
.cloneDestination
:
3606 sys
.stderr
.write("Must specify destination for --keep-path\n")
3611 if not self
.cloneDestination
and len(depotPaths
) > 1:
3612 self
.cloneDestination
= depotPaths
[-1]
3613 depotPaths
= depotPaths
[:-1]
3615 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3616 for p
in depotPaths
:
3617 if not p
.startswith("//"):
3618 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3621 if not self
.cloneDestination
:
3622 self
.cloneDestination
= self
.defaultDestination(args
)
3624 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3626 if not os
.path
.exists(self
.cloneDestination
):
3627 os
.makedirs(self
.cloneDestination
)
3628 chdir(self
.cloneDestination
)
3630 init_cmd
= [ "git", "init" ]
3632 init_cmd
.append("--bare")
3633 retcode
= subprocess
.call(init_cmd
)
3635 raise CalledProcessError(retcode
, init_cmd
)
3637 if not P4Sync
.run(self
, depotPaths
):
3640 # create a master branch and check out a work tree
3641 if gitBranchExists(self
.branch
):
3642 system([ "git", "branch", "master", self
.branch
])
3643 if not self
.cloneBare
:
3644 system([ "git", "checkout", "-f" ])
3646 print 'Not checking out any branch, use ' \
3647 '"git checkout -q -b master <branch>"'
3649 # auto-set this variable if invoked with --use-client-spec
3650 if self
.useClientSpec_from_options
:
3651 system("git config --bool git-p4.useclientspec true")
3655 class P4Branches(Command
):
3657 Command
.__init
__(self
)
3659 self
.description
= ("Shows the git branches that hold imports and their "
3660 + "corresponding perforce depot paths")
3661 self
.verbose
= False
3663 def run(self
, args
):
3664 if originP4BranchesExist():
3665 createOrUpdateBranchesFromOrigin()
3667 cmdline
= "git rev-parse --symbolic "
3668 cmdline
+= " --remotes"
3670 for line
in read_pipe_lines(cmdline
):
3673 if not line
.startswith('p4/') or line
== "p4/HEAD":
3677 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3678 settings
= extractSettingsGitLog(log
)
3680 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3683 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3685 optparse
.IndentedHelpFormatter
.__init
__(self
)
3687 def format_description(self
, description
):
3689 return description
+ "\n"
3693 def printUsage(commands
):
3694 print "usage: %s <command> [options]" % sys
.argv
[0]
3696 print "valid commands: %s" % ", ".join(commands
)
3698 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3703 "submit" : P4Submit
,
3704 "commit" : P4Submit
,
3706 "rebase" : P4Rebase
,
3708 "rollback" : P4RollBack
,
3709 "branches" : P4Branches
3714 if len(sys
.argv
[1:]) == 0:
3715 printUsage(commands
.keys())
3718 cmdName
= sys
.argv
[1]
3720 klass
= commands
[cmdName
]
3723 print "unknown command %s" % cmdName
3725 printUsage(commands
.keys())
3728 options
= cmd
.options
3729 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3733 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3735 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3737 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3739 description
= cmd
.description
,
3740 formatter
= HelpFormatter())
3742 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3744 verbose
= cmd
.verbose
3746 if cmd
.gitdir
== None:
3747 cmd
.gitdir
= os
.path
.abspath(".git")
3748 if not isValidGitDir(cmd
.gitdir
):
3749 # "rev-parse --git-dir" without arguments will try $PWD/.git
3750 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3751 if os
.path
.exists(cmd
.gitdir
):
3752 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3756 if not isValidGitDir(cmd
.gitdir
):
3757 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3758 cmd
.gitdir
+= "/.git"
3760 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3762 # so git commands invoked from the P4 workspace will succeed
3763 os
.environ
["GIT_DIR"] = cmd
.gitdir
3765 if not cmd
.run(args
):
3770 if __name__
== '__main__':