3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess
import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self
, returncode
, cmd
):
39 self
.returncode
= returncode
42 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize
= 512
52 def p4_build_cmd(cmd
):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user
= gitConfig("git-p4.user")
63 real_cmd
+= ["-u",user
]
65 password
= gitConfig("git-p4.password")
67 real_cmd
+= ["-P", password
]
69 port
= gitConfig("git-p4.port")
71 real_cmd
+= ["-p", port
]
73 host
= gitConfig("git-p4.host")
75 real_cmd
+= ["-H", host
]
77 client
= gitConfig("git-p4.client")
79 real_cmd
+= ["-c", client
]
82 if isinstance(cmd
,basestring
):
83 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
88 def chdir(path
, is_client_path
=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
106 if not is_client_path
:
108 os
.environ
['PWD'] = path
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform
.system() == 'Windows':
113 free_bytes
= ctypes
.c_ulonglong(0)
114 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
115 return free_bytes
.value
117 st
= os
.statvfs(os
.getcwd())
118 return st
.f_bavail
* st
.f_frsize
124 sys
.stderr
.write(msg
+ "\n")
127 def write_pipe(c
, stdin
):
129 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
131 expand
= isinstance(c
,basestring
)
132 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
134 val
= pipe
.write(stdin
)
137 die('Command failed: %s' % str(c
))
141 def p4_write_pipe(c
, stdin
):
142 real_cmd
= p4_build_cmd(c
)
143 return write_pipe(real_cmd
, stdin
)
145 def read_pipe(c
, ignore_error
=False):
147 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
149 expand
= isinstance(c
,basestring
)
150 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
151 (out
, err
) = p
.communicate()
152 if p
.returncode
!= 0 and not ignore_error
:
153 die('Command failed: %s\nError: %s' % (str(c
), err
))
156 def p4_read_pipe(c
, ignore_error
=False):
157 real_cmd
= p4_build_cmd(c
)
158 return read_pipe(real_cmd
, ignore_error
)
160 def read_pipe_lines(c
):
162 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
164 expand
= isinstance(c
, basestring
)
165 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
167 val
= pipe
.readlines()
168 if pipe
.close() or p
.wait():
169 die('Command failed: %s' % str(c
))
173 def p4_read_pipe_lines(c
):
174 """Specifically invoke p4 on the command supplied. """
175 real_cmd
= p4_build_cmd(c
)
176 return read_pipe_lines(real_cmd
)
178 def p4_has_command(cmd
):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd
= p4_build_cmd(["help", cmd
])
182 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
183 stderr
=subprocess
.PIPE
)
185 return p
.returncode
== 0
187 def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
193 if not p4_has_command("move"):
195 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
196 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
197 (out
, err
) = p
.communicate()
198 # return code will be 1 in either case
199 if err
.find("Invalid option") >= 0:
201 if err
.find("disabled") >= 0:
203 # assume it failed because @... was invalid changelist
206 def system(cmd
, ignore_error
=False):
207 expand
= isinstance(cmd
,basestring
)
209 sys
.stderr
.write("executing %s\n" % str(cmd
))
210 retcode
= subprocess
.call(cmd
, shell
=expand
)
211 if retcode
and not ignore_error
:
212 raise CalledProcessError(retcode
, cmd
)
217 """Specifically invoke p4 as the system command. """
218 real_cmd
= p4_build_cmd(cmd
)
219 expand
= isinstance(real_cmd
, basestring
)
220 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
222 raise CalledProcessError(retcode
, real_cmd
)
224 _p4_version_string
= None
225 def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
234 global _p4_version_string
235 if not _p4_version_string
:
236 a
= p4_read_pipe_lines(["-V"])
237 _p4_version_string
= a
[-1].rstrip()
238 return _p4_version_string
240 def p4_integrate(src
, dest
):
241 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
243 def p4_sync(f
, *options
):
244 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
247 # forcibly add file names with wildcards
248 if wildcard_present(f
):
249 p4_system(["add", "-f", f
])
251 p4_system(["add", f
])
254 p4_system(["delete", wildcard_encode(f
)])
256 def p4_edit(f
, *options
):
257 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
260 p4_system(["revert", wildcard_encode(f
)])
262 def p4_reopen(type, f
):
263 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
265 def p4_reopen_in_change(changelist
, files
):
266 cmd
= ["reopen", "-c", str(changelist
)] + files
269 def p4_move(src
, dest
):
270 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
272 def p4_last_change():
273 results
= p4CmdList(["changes", "-m", "1"])
274 return int(results
[0]['change'])
276 def p4_describe(change
):
277 """Make sure it returns a valid result by checking for
278 the presence of field "time". Return a dict of the
281 ds
= p4CmdList(["describe", "-s", str(change
)])
283 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
287 if "p4ExitCode" in d
:
288 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
291 if d
["code"] == "error":
292 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
295 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
300 # Canonicalize the p4 type and return a tuple of the
301 # base type, plus any modifiers. See "p4 help filetypes"
302 # for a list and explanation.
304 def split_p4_type(p4type
):
306 p4_filetypes_historical
= {
307 "ctempobj": "binary+Sw",
313 "tempobj": "binary+FSw",
314 "ubinary": "binary+F",
315 "uresource": "resource+F",
316 "uxbinary": "binary+Fx",
317 "xbinary": "binary+x",
319 "xtempobj": "binary+Swx",
321 "xunicode": "unicode+x",
324 if p4type
in p4_filetypes_historical
:
325 p4type
= p4_filetypes_historical
[p4type
]
327 s
= p4type
.split("+")
335 # return the raw p4 type of a file (text, text+ko, etc)
338 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
339 return results
[0]['headType']
342 # Given a type base and modifier, return a regexp matching
343 # the keywords that can be expanded in the file
345 def p4_keywords_regexp_for_type(base
, type_mods
):
346 if base
in ("text", "unicode", "binary"):
348 if "ko" in type_mods
:
350 elif "k" in type_mods
:
351 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
355 \$ # Starts with a dollar, followed by...
356 (%s) # one of the keywords, followed by...
357 (:[^$\n]+)? # possibly an old expansion, followed by...
365 # Given a file, return a regexp matching the possible
366 # RCS keywords that will be expanded, or None for files
367 # with kw expansion turned off.
369 def p4_keywords_regexp_for_file(file):
370 if not os
.path
.exists(file):
373 (type_base
, type_mods
) = split_p4_type(p4_type(file))
374 return p4_keywords_regexp_for_type(type_base
, type_mods
)
376 def setP4ExecBit(file, mode
):
377 # Reopens an already open file and changes the execute bit to match
378 # the execute bit setting in the passed in mode.
382 if not isModeExec(mode
):
383 p4Type
= getP4OpenedType(file)
384 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
385 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
386 if p4Type
[-1] == "+":
387 p4Type
= p4Type
[0:-1]
389 p4_reopen(p4Type
, file)
391 def getP4OpenedType(file):
392 # Returns the perforce file type for the given file.
394 result
= p4_read_pipe(["opened", wildcard_encode(file)])
395 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
397 return match
.group(1)
399 die("Could not determine file type for %s (result: '%s')" % (file, result
))
401 # Return the set of all p4 labels
402 def getP4Labels(depotPaths
):
404 if isinstance(depotPaths
,basestring
):
405 depotPaths
= [depotPaths
]
407 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
413 # Return the set of all git tags
416 for line
in read_pipe_lines(["git", "tag"]):
421 def diffTreePattern():
422 # This is a simple generator for the diff tree regex pattern. This could be
423 # a class variable if this and parseDiffTreeEntry were a part of a class.
424 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
428 def parseDiffTreeEntry(entry
):
429 """Parses a single diff tree entry into its component elements.
431 See git-diff-tree(1) manpage for details about the format of the diff
432 output. This method returns a dictionary with the following elements:
434 src_mode - The mode of the source file
435 dst_mode - The mode of the destination file
436 src_sha1 - The sha1 for the source file
437 dst_sha1 - The sha1 fr the destination file
438 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
439 status_score - The score for the status (applicable for 'C' and 'R'
440 statuses). This is None if there is no score.
441 src - The path for the source file.
442 dst - The path for the destination file. This is only present for
443 copy or renames. If it is not present, this is None.
445 If the pattern is not matched, None is returned."""
447 match
= diffTreePattern().next().match(entry
)
450 'src_mode': match
.group(1),
451 'dst_mode': match
.group(2),
452 'src_sha1': match
.group(3),
453 'dst_sha1': match
.group(4),
454 'status': match
.group(5),
455 'status_score': match
.group(6),
456 'src': match
.group(7),
457 'dst': match
.group(10)
461 def isModeExec(mode
):
462 # Returns True if the given git mode represents an executable file,
464 return mode
[-3:] == "755"
466 def isModeExecChanged(src_mode
, dst_mode
):
467 return isModeExec(src_mode
) != isModeExec(dst_mode
)
469 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
471 if isinstance(cmd
,basestring
):
478 cmd
= p4_build_cmd(cmd
)
480 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
482 # Use a temporary file to avoid deadlocks without
483 # subprocess.communicate(), which would put another copy
484 # of stdout into memory.
486 if stdin
is not None:
487 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
488 if isinstance(stdin
,basestring
):
489 stdin_file
.write(stdin
)
492 stdin_file
.write(i
+ '\n')
496 p4
= subprocess
.Popen(cmd
,
499 stdout
=subprocess
.PIPE
)
504 entry
= marshal
.load(p4
.stdout
)
514 entry
["p4ExitCode"] = exitCode
520 list = p4CmdList(cmd
)
526 def p4Where(depotPath
):
527 if not depotPath
.endswith("/"):
529 depotPathLong
= depotPath
+ "..."
530 outputList
= p4CmdList(["where", depotPathLong
])
532 for entry
in outputList
:
533 if "depotFile" in entry
:
534 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
535 # The base path always ends with "/...".
536 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
539 elif "data" in entry
:
540 data
= entry
.get("data")
541 space
= data
.find(" ")
542 if data
[:space
] == depotPath
:
547 if output
["code"] == "error":
551 clientPath
= output
.get("path")
552 elif "data" in output
:
553 data
= output
.get("data")
554 lastSpace
= data
.rfind(" ")
555 clientPath
= data
[lastSpace
+ 1:]
557 if clientPath
.endswith("..."):
558 clientPath
= clientPath
[:-3]
561 def currentGitBranch():
562 retcode
= system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error
=True)
567 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
569 def isValidGitDir(path
):
570 if (os
.path
.exists(path
+ "/HEAD")
571 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
575 def parseRevision(ref
):
576 return read_pipe("git rev-parse %s" % ref
).strip()
578 def branchExists(ref
):
579 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
583 def extractLogMessageFromGitCommit(commit
):
586 ## fixme: title is first line of commit, not 1st paragraph.
588 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
597 def extractSettingsGitLog(log
):
599 for line
in log
.split("\n"):
601 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
605 assignments
= m
.group(1).split (':')
606 for a
in assignments
:
608 key
= vals
[0].strip()
609 val
= ('='.join (vals
[1:])).strip()
610 if val
.endswith ('\"') and val
.startswith('"'):
615 paths
= values
.get("depot-paths")
617 paths
= values
.get("depot-path")
619 values
['depot-paths'] = paths
.split(',')
622 def gitBranchExists(branch
):
623 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
624 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
625 return proc
.wait() == 0;
629 def gitConfig(key
, typeSpecifier
=None):
630 if not _gitConfig
.has_key(key
):
631 cmd
= [ "git", "config" ]
633 cmd
+= [ typeSpecifier
]
635 s
= read_pipe(cmd
, ignore_error
=True)
636 _gitConfig
[key
] = s
.strip()
637 return _gitConfig
[key
]
639 def gitConfigBool(key
):
640 """Return a bool, using git config --bool. It is True only if the
641 variable is set to true, and False if set to false or not present
644 if not _gitConfig
.has_key(key
):
645 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
646 return _gitConfig
[key
]
648 def gitConfigInt(key
):
649 if not _gitConfig
.has_key(key
):
650 cmd
= [ "git", "config", "--int", key
]
651 s
= read_pipe(cmd
, ignore_error
=True)
654 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
656 _gitConfig
[key
] = None
657 return _gitConfig
[key
]
659 def gitConfigList(key
):
660 if not _gitConfig
.has_key(key
):
661 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
662 _gitConfig
[key
] = s
.strip().split(os
.linesep
)
663 if _gitConfig
[key
] == ['']:
665 return _gitConfig
[key
]
667 def p4BranchesInGit(branchesAreInRemotes
=True):
668 """Find all the branches whose names start with "p4/", looking
669 in remotes or heads as specified by the argument. Return
670 a dictionary of { branch: revision } for each one found.
671 The branch names are the short names, without any
676 cmdline
= "git rev-parse --symbolic "
677 if branchesAreInRemotes
:
678 cmdline
+= "--remotes"
680 cmdline
+= "--branches"
682 for line
in read_pipe_lines(cmdline
):
686 if not line
.startswith('p4/'):
688 # special symbolic ref to p4/master
689 if line
== "p4/HEAD":
692 # strip off p4/ prefix
693 branch
= line
[len("p4/"):]
695 branches
[branch
] = parseRevision(line
)
699 def branch_exists(branch
):
700 """Make sure that the given ref name really exists."""
702 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
703 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
704 out
, _
= p
.communicate()
707 # expect exactly one line of output: the branch name
708 return out
.rstrip() == branch
710 def findUpstreamBranchPoint(head
= "HEAD"):
711 branches
= p4BranchesInGit()
712 # map from depot-path to branch name
713 branchByDepotPath
= {}
714 for branch
in branches
.keys():
715 tip
= branches
[branch
]
716 log
= extractLogMessageFromGitCommit(tip
)
717 settings
= extractSettingsGitLog(log
)
718 if settings
.has_key("depot-paths"):
719 paths
= ",".join(settings
["depot-paths"])
720 branchByDepotPath
[paths
] = "remotes/p4/" + branch
724 while parent
< 65535:
725 commit
= head
+ "~%s" % parent
726 log
= extractLogMessageFromGitCommit(commit
)
727 settings
= extractSettingsGitLog(log
)
728 if settings
.has_key("depot-paths"):
729 paths
= ",".join(settings
["depot-paths"])
730 if branchByDepotPath
.has_key(paths
):
731 return [branchByDepotPath
[paths
], settings
]
735 return ["", settings
]
737 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
739 print ("Creating/updating branch(es) in %s based on origin branch(es)"
742 originPrefix
= "origin/p4/"
744 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
746 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
749 headName
= line
[len(originPrefix
):]
750 remoteHead
= localRefPrefix
+ headName
753 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
754 if (not original
.has_key('depot-paths')
755 or not original
.has_key('change')):
759 if not gitBranchExists(remoteHead
):
761 print "creating %s" % remoteHead
764 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
765 if settings
.has_key('change') > 0:
766 if settings
['depot-paths'] == original
['depot-paths']:
767 originP4Change
= int(original
['change'])
768 p4Change
= int(settings
['change'])
769 if originP4Change
> p4Change
:
770 print ("%s (%s) is newer than %s (%s). "
771 "Updating p4 branch from origin."
772 % (originHead
, originP4Change
,
773 remoteHead
, p4Change
))
776 print ("Ignoring: %s was imported from %s while "
777 "%s was imported from %s"
778 % (originHead
, ','.join(original
['depot-paths']),
779 remoteHead
, ','.join(settings
['depot-paths'])))
782 system("git update-ref %s %s" % (remoteHead
, originHead
))
784 def originP4BranchesExist():
785 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
788 def p4ParseNumericChangeRange(parts
):
789 changeStart
= int(parts
[0][1:])
790 if parts
[1] == '#head':
791 changeEnd
= p4_last_change()
793 changeEnd
= int(parts
[1])
795 return (changeStart
, changeEnd
)
797 def chooseBlockSize(blockSize
):
801 return defaultBlockSize
803 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
806 # Parse the change range into start and end. Try to find integer
807 # revision ranges as these can be broken up into blocks to avoid
808 # hitting server-side limits (maxrows, maxscanresults). But if
809 # that doesn't work, fall back to using the raw revision specifier
810 # strings, without using block mode.
812 if changeRange
is None or changeRange
== '':
814 changeEnd
= p4_last_change()
815 block_size
= chooseBlockSize(requestedBlockSize
)
817 parts
= changeRange
.split(',')
818 assert len(parts
) == 2
820 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
821 block_size
= chooseBlockSize(requestedBlockSize
)
823 changeStart
= parts
[0][1:]
825 if requestedBlockSize
:
826 die("cannot use --changes-block-size with non-numeric revisions")
831 # Retrieve changes a block at a time, to prevent running
832 # into a MaxResults/MaxScanRows error from the server.
838 end
= min(changeEnd
, changeStart
+ block_size
)
839 revisionRange
= "%d,%d" % (changeStart
, end
)
841 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
844 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
846 # Insert changes in chronological order
847 for line
in reversed(p4_read_pipe_lines(cmd
)):
848 changes
.append(int(line
.split(" ")[1]))
856 changeStart
= end
+ 1
858 changes
= sorted(changes
)
861 def p4PathStartsWith(path
, prefix
):
862 # This method tries to remedy a potential mixed-case issue:
864 # If UserA adds //depot/DirA/file1
865 # and UserB adds //depot/dira/file2
867 # we may or may not have a problem. If you have core.ignorecase=true,
868 # we treat DirA and dira as the same directory
869 if gitConfigBool("core.ignorecase"):
870 return path
.lower().startswith(prefix
.lower())
871 return path
.startswith(prefix
)
874 """Look at the p4 client spec, create a View() object that contains
875 all the mappings, and return it."""
877 specList
= p4CmdList("client -o")
878 if len(specList
) != 1:
879 die('Output from "client -o" is %d lines, expecting 1' %
882 # dictionary of all client parameters
886 client_name
= entry
["Client"]
888 # just the keys that start with "View"
889 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
892 view
= View(client_name
)
894 # append the lines, in order, to the view
895 for view_num
in range(len(view_keys
)):
896 k
= "View%d" % view_num
897 if k
not in view_keys
:
898 die("Expected view key %s missing" % k
)
899 view
.append(entry
[k
])
904 """Grab the client directory."""
906 output
= p4CmdList("client -o")
908 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
911 if "Root" not in entry
:
912 die('Client has no "Root"')
917 # P4 wildcards are not allowed in filenames. P4 complains
918 # if you simply add them, but you can force it with "-f", in
919 # which case it translates them into %xx encoding internally.
921 def wildcard_decode(path
):
922 # Search for and fix just these four characters. Do % last so
923 # that fixing it does not inadvertently create new %-escapes.
924 # Cannot have * in a filename in windows; untested as to
925 # what p4 would do in such a case.
926 if not platform
.system() == "Windows":
927 path
= path
.replace("%2A", "*")
928 path
= path
.replace("%23", "#") \
929 .replace("%40", "@") \
933 def wildcard_encode(path
):
934 # do % first to avoid double-encoding the %s introduced here
935 path
= path
.replace("%", "%25") \
936 .replace("*", "%2A") \
937 .replace("#", "%23") \
941 def wildcard_present(path
):
942 m
= re
.search("[*#@%]", path
)
945 class LargeFileSystem(object):
946 """Base class for large file system support."""
948 def __init__(self
, writeToGitStream
):
949 self
.largeFiles
= set()
950 self
.writeToGitStream
= writeToGitStream
952 def generatePointer(self
, cloneDestination
, contentFile
):
953 """Return the content of a pointer file that is stored in Git instead of
954 the actual content."""
955 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
957 def pushFile(self
, localLargeFile
):
958 """Push the actual content which is not stored in the Git repository to
960 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
962 def hasLargeFileExtension(self
, relPath
):
965 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
969 def generateTempFile(self
, contents
):
970 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
974 return contentFile
.name
976 def exceedsLargeFileThreshold(self
, relPath
, contents
):
977 if gitConfigInt('git-p4.largeFileThreshold'):
978 contentsSize
= sum(len(d
) for d
in contents
)
979 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
981 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
982 contentsSize
= sum(len(d
) for d
in contents
)
983 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
985 contentTempFile
= self
.generateTempFile(contents
)
986 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
987 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
988 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
990 compressedContentsSize
= zf
.infolist()[0].compress_size
991 os
.remove(contentTempFile
)
992 os
.remove(compressedContentFile
.name
)
993 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
997 def addLargeFile(self
, relPath
):
998 self
.largeFiles
.add(relPath
)
1000 def removeLargeFile(self
, relPath
):
1001 self
.largeFiles
.remove(relPath
)
1003 def isLargeFile(self
, relPath
):
1004 return relPath
in self
.largeFiles
1006 def processContent(self
, git_mode
, relPath
, contents
):
1007 """Processes the content of git fast import. This method decides if a
1008 file is stored in the large file system and handles all necessary
1010 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1011 contentTempFile
= self
.generateTempFile(contents
)
1012 (git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1014 # Move temp file to final location in large file system
1015 largeFileDir
= os
.path
.dirname(localLargeFile
)
1016 if not os
.path
.isdir(largeFileDir
):
1017 os
.makedirs(largeFileDir
)
1018 shutil
.move(contentTempFile
, localLargeFile
)
1019 self
.addLargeFile(relPath
)
1020 if gitConfigBool('git-p4.largeFilePush'):
1021 self
.pushFile(localLargeFile
)
1023 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1024 return (git_mode
, contents
)
1026 class MockLFS(LargeFileSystem
):
1027 """Mock large file system for testing."""
1029 def generatePointer(self
, contentFile
):
1030 """The pointer content is the original content prefixed with "pointer-".
1031 The local filename of the large file storage is derived from the file content.
1033 with
open(contentFile
, 'r') as f
:
1036 pointerContents
= 'pointer-' + content
1037 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1038 return (gitMode
, pointerContents
, localLargeFile
)
1040 def pushFile(self
, localLargeFile
):
1041 """The remote filename of the large file storage is the same as the local
1042 one but in a different directory.
1044 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1045 if not os
.path
.exists(remotePath
):
1046 os
.makedirs(remotePath
)
1047 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1049 class GitLFS(LargeFileSystem
):
1050 """Git LFS as backend for the git-p4 large file system.
1051 See https://git-lfs.github.com/ for details."""
1053 def __init__(self
, *args
):
1054 LargeFileSystem
.__init
__(self
, *args
)
1055 self
.baseGitAttributes
= []
1057 def generatePointer(self
, contentFile
):
1058 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1059 mode and content which is stored in the Git repository instead of
1060 the actual content. Return also the new location of the actual
1063 pointerProcess
= subprocess
.Popen(
1064 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1065 stdout
=subprocess
.PIPE
1067 pointerFile
= pointerProcess
.stdout
.read()
1068 if pointerProcess
.wait():
1069 os
.remove(contentFile
)
1070 die('git-lfs pointer command failed. Did you install the extension?')
1072 # Git LFS removed the preamble in the output of the 'pointer' command
1073 # starting from version 1.2.0. Check for the preamble here to support
1075 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1076 if pointerFile
.startswith('Git LFS pointer for'):
1077 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1079 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1080 localLargeFile
= os
.path
.join(
1082 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1085 # LFS Spec states that pointer files should not have the executable bit set.
1087 return (gitMode
, pointerFile
, localLargeFile
)
1089 def pushFile(self
, localLargeFile
):
1090 uploadProcess
= subprocess
.Popen(
1091 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1093 if uploadProcess
.wait():
1094 die('git-lfs push command failed. Did you define a remote?')
1096 def generateGitAttributes(self
):
1098 self
.baseGitAttributes
+
1102 '# Git LFS (see https://git-lfs.github.com/)\n',
1105 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1106 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1108 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1109 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1113 def addLargeFile(self
, relPath
):
1114 LargeFileSystem
.addLargeFile(self
, relPath
)
1115 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1117 def removeLargeFile(self
, relPath
):
1118 LargeFileSystem
.removeLargeFile(self
, relPath
)
1119 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1121 def processContent(self
, git_mode
, relPath
, contents
):
1122 if relPath
== '.gitattributes':
1123 self
.baseGitAttributes
= contents
1124 return (git_mode
, self
.generateGitAttributes())
1126 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1130 self
.usage
= "usage: %prog [options]"
1131 self
.needsGit
= True
1132 self
.verbose
= False
1136 self
.userMapFromPerforceServer
= False
1137 self
.myP4UserId
= None
1141 return self
.myP4UserId
1143 results
= p4CmdList("user -o")
1145 if r
.has_key('User'):
1146 self
.myP4UserId
= r
['User']
1148 die("Could not find your p4 user id")
1150 def p4UserIsMe(self
, p4User
):
1151 # return True if the given p4 user is actually me
1152 me
= self
.p4UserId()
1153 if not p4User
or p4User
!= me
:
1158 def getUserCacheFilename(self
):
1159 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1160 return home
+ "/.gitp4-usercache.txt"
1162 def getUserMapFromPerforceServer(self
):
1163 if self
.userMapFromPerforceServer
:
1168 for output
in p4CmdList("users"):
1169 if not output
.has_key("User"):
1171 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1172 self
.emails
[output
["Email"]] = output
["User"]
1174 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1175 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1176 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1177 if mapUser
and len(mapUser
[0]) == 3:
1178 user
= mapUser
[0][0]
1179 fullname
= mapUser
[0][1]
1180 email
= mapUser
[0][2]
1181 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1182 self
.emails
[email
] = user
1185 for (key
, val
) in self
.users
.items():
1186 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1188 open(self
.getUserCacheFilename(), "wb").write(s
)
1189 self
.userMapFromPerforceServer
= True
1191 def loadUserMapFromCache(self
):
1193 self
.userMapFromPerforceServer
= False
1195 cache
= open(self
.getUserCacheFilename(), "rb")
1196 lines
= cache
.readlines()
1199 entry
= line
.strip().split("\t")
1200 self
.users
[entry
[0]] = entry
[1]
1202 self
.getUserMapFromPerforceServer()
1204 class P4Debug(Command
):
1206 Command
.__init
__(self
)
1208 self
.description
= "A tool to debug the output of p4 -G."
1209 self
.needsGit
= False
1211 def run(self
, args
):
1213 for output
in p4CmdList(args
):
1214 print 'Element: %d' % j
1219 class P4RollBack(Command
):
1221 Command
.__init
__(self
)
1223 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1225 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1226 self
.rollbackLocalBranches
= False
1228 def run(self
, args
):
1231 maxChange
= int(args
[0])
1233 if "p4ExitCode" in p4Cmd("changes -m 1"):
1234 die("Problems executing p4");
1236 if self
.rollbackLocalBranches
:
1237 refPrefix
= "refs/heads/"
1238 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1240 refPrefix
= "refs/remotes/"
1241 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1244 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1246 ref
= refPrefix
+ line
1247 log
= extractLogMessageFromGitCommit(ref
)
1248 settings
= extractSettingsGitLog(log
)
1250 depotPaths
= settings
['depot-paths']
1251 change
= settings
['change']
1255 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1256 for p
in depotPaths
]))) == 0:
1257 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1258 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1261 while change
and int(change
) > maxChange
:
1264 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1265 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1266 log
= extractLogMessageFromGitCommit(ref
)
1267 settings
= extractSettingsGitLog(log
)
1270 depotPaths
= settings
['depot-paths']
1271 change
= settings
['change']
1274 print "%s rewound to %s" % (ref
, change
)
1278 class P4Submit(Command
, P4UserMap
):
1280 conflict_behavior_choices
= ("ask", "skip", "quit")
1283 Command
.__init
__(self
)
1284 P4UserMap
.__init
__(self
)
1286 optparse
.make_option("--origin", dest
="origin"),
1287 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1288 # preserve the user, requires relevant p4 permissions
1289 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1290 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1291 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1292 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1293 optparse
.make_option("--conflict", dest
="conflict_behavior",
1294 choices
=self
.conflict_behavior_choices
),
1295 optparse
.make_option("--branch", dest
="branch"),
1296 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1297 help="Shelve instead of submit. Shelved files are reverted, "
1298 "restoring the workspace to the state before the shelve"),
1299 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="store", type="int",
1300 metavar
="CHANGELIST",
1301 help="update an existing shelved changelist, implies --shelve")
1303 self
.description
= "Submit changes from git to the perforce depot."
1304 self
.usage
+= " [name of git branch to submit into perforce depot]"
1306 self
.detectRenames
= False
1307 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1308 self
.dry_run
= False
1310 self
.update_shelve
= None
1311 self
.prepare_p4_only
= False
1312 self
.conflict_behavior
= None
1313 self
.isWindows
= (platform
.system() == "Windows")
1314 self
.exportLabels
= False
1315 self
.p4HasMoveCommand
= p4_has_move_command()
1318 if gitConfig('git-p4.largeFileSystem'):
1319 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1322 if len(p4CmdList("opened ...")) > 0:
1323 die("You have files opened with perforce! Close them before starting the sync.")
1325 def separate_jobs_from_description(self
, message
):
1326 """Extract and return a possible Jobs field in the commit
1327 message. It goes into a separate section in the p4 change
1330 A jobs line starts with "Jobs:" and looks like a new field
1331 in a form. Values are white-space separated on the same
1332 line or on following lines that start with a tab.
1334 This does not parse and extract the full git commit message
1335 like a p4 form. It just sees the Jobs: line as a marker
1336 to pass everything from then on directly into the p4 form,
1337 but outside the description section.
1339 Return a tuple (stripped log message, jobs string)."""
1341 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1343 return (message
, None)
1345 jobtext
= message
[m
.start():]
1346 stripped_message
= message
[:m
.start()].rstrip()
1347 return (stripped_message
, jobtext
)
1349 def prepareLogMessage(self
, template
, message
, jobs
):
1350 """Edits the template returned from "p4 change -o" to insert
1351 the message in the Description field, and the jobs text in
1355 inDescriptionSection
= False
1357 for line
in template
.split("\n"):
1358 if line
.startswith("#"):
1359 result
+= line
+ "\n"
1362 if inDescriptionSection
:
1363 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1364 inDescriptionSection
= False
1365 # insert Jobs section
1367 result
+= jobs
+ "\n"
1371 if line
.startswith("Description:"):
1372 inDescriptionSection
= True
1374 for messageLine
in message
.split("\n"):
1375 line
+= "\t" + messageLine
+ "\n"
1377 result
+= line
+ "\n"
1381 def patchRCSKeywords(self
, file, pattern
):
1382 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1383 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1385 outFile
= os
.fdopen(handle
, "w+")
1386 inFile
= open(file, "r")
1387 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1388 for line
in inFile
.readlines():
1389 line
= regexp
.sub(r
'$\1$', line
)
1393 # Forcibly overwrite the original file
1395 shutil
.move(outFileName
, file)
1397 # cleanup our temporary file
1398 os
.unlink(outFileName
)
1399 print "Failed to strip RCS keywords in %s" % file
1402 print "Patched up RCS keywords in %s" % file
1404 def p4UserForCommit(self
,id):
1405 # Return the tuple (perforce user,git email) for a given git commit id
1406 self
.getUserMapFromPerforceServer()
1407 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1408 "--format=%ae", id])
1409 gitEmail
= gitEmail
.strip()
1410 if not self
.emails
.has_key(gitEmail
):
1411 return (None,gitEmail
)
1413 return (self
.emails
[gitEmail
],gitEmail
)
1415 def checkValidP4Users(self
,commits
):
1416 # check if any git authors cannot be mapped to p4 users
1418 (user
,email
) = self
.p4UserForCommit(id)
1420 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1421 if gitConfigBool("git-p4.allowMissingP4Users"):
1424 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1426 def lastP4Changelist(self
):
1427 # Get back the last changelist number submitted in this client spec. This
1428 # then gets used to patch up the username in the change. If the same
1429 # client spec is being used by multiple processes then this might go
1431 results
= p4CmdList("client -o") # find the current client
1434 if r
.has_key('Client'):
1435 client
= r
['Client']
1438 die("could not get client spec")
1439 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1441 if r
.has_key('change'):
1443 die("Could not get changelist number for last submit - cannot patch up user details")
1445 def modifyChangelistUser(self
, changelist
, newUser
):
1446 # fixup the user field of a changelist after it has been submitted.
1447 changes
= p4CmdList("change -o %s" % changelist
)
1448 if len(changes
) != 1:
1449 die("Bad output from p4 change modifying %s to user %s" %
1450 (changelist
, newUser
))
1453 if c
['User'] == newUser
: return # nothing to do
1455 input = marshal
.dumps(c
)
1457 result
= p4CmdList("change -f -i", stdin
=input)
1459 if r
.has_key('code'):
1460 if r
['code'] == 'error':
1461 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1462 if r
.has_key('data'):
1463 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1465 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1467 def canChangeChangelists(self
):
1468 # check to see if we have p4 admin or super-user permissions, either of
1469 # which are required to modify changelists.
1470 results
= p4CmdList(["protects", self
.depotPath
])
1472 if r
.has_key('perm'):
1473 if r
['perm'] == 'admin':
1475 if r
['perm'] == 'super':
1479 def prepareSubmitTemplate(self
, changelist
=None):
1480 """Run "p4 change -o" to grab a change specification template.
1481 This does not use "p4 -G", as it is nice to keep the submission
1482 template in original order, since a human might edit it.
1484 Remove lines in the Files section that show changes to files
1485 outside the depot path we're committing into."""
1487 [upstream
, settings
] = findUpstreamBranchPoint()
1490 inFilesSection
= False
1491 args
= ['change', '-o']
1493 args
.append(str(changelist
))
1495 for line
in p4_read_pipe_lines(args
):
1496 if line
.endswith("\r\n"):
1497 line
= line
[:-2] + "\n"
1499 if line
.startswith("\t"):
1500 # path starts and ends with a tab
1502 lastTab
= path
.rfind("\t")
1504 path
= path
[:lastTab
]
1505 if settings
.has_key('depot-paths'):
1506 if not [p
for p
in settings
['depot-paths']
1507 if p4PathStartsWith(path
, p
)]:
1510 if not p4PathStartsWith(path
, self
.depotPath
):
1513 inFilesSection
= False
1515 if line
.startswith("Files:"):
1516 inFilesSection
= True
1522 def edit_template(self
, template_file
):
1523 """Invoke the editor to let the user change the submission
1524 message. Return true if okay to continue with the submit."""
1526 # if configured to skip the editing part, just submit
1527 if gitConfigBool("git-p4.skipSubmitEdit"):
1530 # look at the modification time, to check later if the user saved
1532 mtime
= os
.stat(template_file
).st_mtime
1535 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1536 editor
= os
.environ
.get("P4EDITOR")
1538 editor
= read_pipe("git var GIT_EDITOR").strip()
1539 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1541 # If the file was not saved, prompt to see if this patch should
1542 # be skipped. But skip this verification step if configured so.
1543 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1546 # modification time updated means user saved the file
1547 if os
.stat(template_file
).st_mtime
> mtime
:
1551 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1557 def get_diff_description(self
, editedFiles
, filesToAdd
):
1559 if os
.environ
.has_key("P4DIFF"):
1560 del(os
.environ
["P4DIFF"])
1562 for editedFile
in editedFiles
:
1563 diff
+= p4_read_pipe(['diff', '-du',
1564 wildcard_encode(editedFile
)])
1568 for newFile
in filesToAdd
:
1569 newdiff
+= "==== new file ====\n"
1570 newdiff
+= "--- /dev/null\n"
1571 newdiff
+= "+++ %s\n" % newFile
1572 f
= open(newFile
, "r")
1573 for line
in f
.readlines():
1574 newdiff
+= "+" + line
1577 return (diff
+ newdiff
).replace('\r\n', '\n')
1579 def applyCommit(self
, id):
1580 """Apply one commit, return True if it succeeded."""
1582 print "Applying", read_pipe(["git", "show", "-s",
1583 "--format=format:%h %s", id])
1585 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1587 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1589 filesToChangeType
= set()
1590 filesToDelete
= set()
1592 pureRenameCopy
= set()
1593 filesToChangeExecBit
= {}
1597 diff
= parseDiffTreeEntry(line
)
1598 modifier
= diff
['status']
1600 all_files
.append(path
)
1604 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1605 filesToChangeExecBit
[path
] = diff
['dst_mode']
1606 editedFiles
.add(path
)
1607 elif modifier
== "A":
1608 filesToAdd
.add(path
)
1609 filesToChangeExecBit
[path
] = diff
['dst_mode']
1610 if path
in filesToDelete
:
1611 filesToDelete
.remove(path
)
1612 elif modifier
== "D":
1613 filesToDelete
.add(path
)
1614 if path
in filesToAdd
:
1615 filesToAdd
.remove(path
)
1616 elif modifier
== "C":
1617 src
, dest
= diff
['src'], diff
['dst']
1618 p4_integrate(src
, dest
)
1619 pureRenameCopy
.add(dest
)
1620 if diff
['src_sha1'] != diff
['dst_sha1']:
1622 pureRenameCopy
.discard(dest
)
1623 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1625 pureRenameCopy
.discard(dest
)
1626 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1628 # turn off read-only attribute
1629 os
.chmod(dest
, stat
.S_IWRITE
)
1631 editedFiles
.add(dest
)
1632 elif modifier
== "R":
1633 src
, dest
= diff
['src'], diff
['dst']
1634 if self
.p4HasMoveCommand
:
1635 p4_edit(src
) # src must be open before move
1636 p4_move(src
, dest
) # opens for (move/delete, move/add)
1638 p4_integrate(src
, dest
)
1639 if diff
['src_sha1'] != diff
['dst_sha1']:
1642 pureRenameCopy
.add(dest
)
1643 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1644 if not self
.p4HasMoveCommand
:
1645 p4_edit(dest
) # with move: already open, writable
1646 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1647 if not self
.p4HasMoveCommand
:
1649 os
.chmod(dest
, stat
.S_IWRITE
)
1651 filesToDelete
.add(src
)
1652 editedFiles
.add(dest
)
1653 elif modifier
== "T":
1654 filesToChangeType
.add(path
)
1656 die("unknown modifier %s for %s" % (modifier
, path
))
1658 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1659 patchcmd
= diffcmd
+ " | git apply "
1660 tryPatchCmd
= patchcmd
+ "--check -"
1661 applyPatchCmd
= patchcmd
+ "--check --apply -"
1662 patch_succeeded
= True
1664 if os
.system(tryPatchCmd
) != 0:
1665 fixed_rcs_keywords
= False
1666 patch_succeeded
= False
1667 print "Unfortunately applying the change failed!"
1669 # Patch failed, maybe it's just RCS keyword woes. Look through
1670 # the patch to see if that's possible.
1671 if gitConfigBool("git-p4.attemptRCSCleanup"):
1675 for file in editedFiles | filesToDelete
:
1676 # did this file's delta contain RCS keywords?
1677 pattern
= p4_keywords_regexp_for_file(file)
1680 # this file is a possibility...look for RCS keywords.
1681 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1682 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1683 if regexp
.search(line
):
1685 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1686 kwfiles
[file] = pattern
1689 for file in kwfiles
:
1691 print "zapping %s with %s" % (line
,pattern
)
1692 # File is being deleted, so not open in p4. Must
1693 # disable the read-only bit on windows.
1694 if self
.isWindows
and file not in editedFiles
:
1695 os
.chmod(file, stat
.S_IWRITE
)
1696 self
.patchRCSKeywords(file, kwfiles
[file])
1697 fixed_rcs_keywords
= True
1699 if fixed_rcs_keywords
:
1700 print "Retrying the patch with RCS keywords cleaned up"
1701 if os
.system(tryPatchCmd
) == 0:
1702 patch_succeeded
= True
1704 if not patch_succeeded
:
1705 for f
in editedFiles
:
1710 # Apply the patch for real, and do add/delete/+x handling.
1712 system(applyPatchCmd
)
1714 for f
in filesToChangeType
:
1715 p4_edit(f
, "-t", "auto")
1716 for f
in filesToAdd
:
1718 for f
in filesToDelete
:
1722 # Set/clear executable bits
1723 for f
in filesToChangeExecBit
.keys():
1724 mode
= filesToChangeExecBit
[f
]
1725 setP4ExecBit(f
, mode
)
1727 if self
.update_shelve
:
1728 print("all_files = %s" % str(all_files
))
1729 p4_reopen_in_change(self
.update_shelve
, all_files
)
1732 # Build p4 change description, starting with the contents
1733 # of the git commit message.
1735 logMessage
= extractLogMessageFromGitCommit(id)
1736 logMessage
= logMessage
.strip()
1737 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1739 template
= self
.prepareSubmitTemplate(self
.update_shelve
)
1740 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1742 if self
.preserveUser
:
1743 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1745 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1746 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1747 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1748 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1750 separatorLine
= "######## everything below this line is just the diff #######\n"
1751 if not self
.prepare_p4_only
:
1752 submitTemplate
+= separatorLine
1753 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
)
1755 (handle
, fileName
) = tempfile
.mkstemp()
1756 tmpFile
= os
.fdopen(handle
, "w+b")
1758 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1759 tmpFile
.write(submitTemplate
)
1762 if self
.prepare_p4_only
:
1764 # Leave the p4 tree prepared, and the submit template around
1765 # and let the user decide what to do next
1768 print "P4 workspace prepared for submission."
1769 print "To submit or revert, go to client workspace"
1770 print " " + self
.clientPath
1772 print "To submit, use \"p4 submit\" to write a new description,"
1773 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1774 " \"git p4\"." % fileName
1775 print "You can delete the file \"%s\" when finished." % fileName
1777 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1778 print "To preserve change ownership by user %s, you must\n" \
1779 "do \"p4 change -f <change>\" after submitting and\n" \
1780 "edit the User field."
1782 print "After submitting, renamed files must be re-synced."
1783 print "Invoke \"p4 sync -f\" on each of these files:"
1784 for f
in pureRenameCopy
:
1788 print "To revert the changes, use \"p4 revert ...\", and delete"
1789 print "the submit template file \"%s\"" % fileName
1791 print "Since the commit adds new files, they must be deleted:"
1792 for f
in filesToAdd
:
1798 # Let the user edit the change description, then submit it.
1803 if self
.edit_template(fileName
):
1804 # read the edited message and submit
1805 tmpFile
= open(fileName
, "rb")
1806 message
= tmpFile
.read()
1809 message
= message
.replace("\r\n", "\n")
1810 submitTemplate
= message
[:message
.index(separatorLine
)]
1812 if self
.update_shelve
:
1813 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
1815 p4_write_pipe(['shelve', '-i'], submitTemplate
)
1817 p4_write_pipe(['submit', '-i'], submitTemplate
)
1818 # The rename/copy happened by applying a patch that created a
1819 # new file. This leaves it writable, which confuses p4.
1820 for f
in pureRenameCopy
:
1823 if self
.preserveUser
:
1825 # Get last changelist number. Cannot easily get it from
1826 # the submit command output as the output is
1828 changelist
= self
.lastP4Changelist()
1829 self
.modifyChangelistUser(changelist
, p4User
)
1835 if not submitted
or self
.shelve
:
1837 print ("Reverting shelved files.")
1839 print ("Submission cancelled, undoing p4 changes.")
1840 for f
in editedFiles | filesToDelete
:
1842 for f
in filesToAdd
:
1849 # Export git tags as p4 labels. Create a p4 label and then tag
1851 def exportGitTags(self
, gitTags
):
1852 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1853 if len(validLabelRegexp
) == 0:
1854 validLabelRegexp
= defaultLabelRegexp
1855 m
= re
.compile(validLabelRegexp
)
1857 for name
in gitTags
:
1859 if not m
.match(name
):
1861 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1864 # Get the p4 commit this corresponds to
1865 logMessage
= extractLogMessageFromGitCommit(name
)
1866 values
= extractSettingsGitLog(logMessage
)
1868 if not values
.has_key('change'):
1869 # a tag pointing to something not sent to p4; ignore
1871 print "git tag %s does not give a p4 commit" % name
1874 changelist
= values
['change']
1876 # Get the tag details.
1880 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1883 if re
.match(r
'tag\s+', l
):
1885 elif re
.match(r
'\s*$', l
):
1892 body
= ["lightweight tag imported by git p4\n"]
1894 # Create the label - use the same view as the client spec we are using
1895 clientSpec
= getClientSpec()
1897 labelTemplate
= "Label: %s\n" % name
1898 labelTemplate
+= "Description:\n"
1900 labelTemplate
+= "\t" + b
+ "\n"
1901 labelTemplate
+= "View:\n"
1902 for depot_side
in clientSpec
.mappings
:
1903 labelTemplate
+= "\t%s\n" % depot_side
1906 print "Would create p4 label %s for tag" % name
1907 elif self
.prepare_p4_only
:
1908 print "Not creating p4 label %s for tag due to option" \
1909 " --prepare-p4-only" % name
1911 p4_write_pipe(["label", "-i"], labelTemplate
)
1914 p4_system(["tag", "-l", name
] +
1915 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
1918 print "created p4 label for tag %s" % name
1920 def run(self
, args
):
1922 self
.master
= currentGitBranch()
1923 elif len(args
) == 1:
1924 self
.master
= args
[0]
1925 if not branchExists(self
.master
):
1926 die("Branch %s does not exist" % self
.master
)
1931 allowSubmit
= gitConfig("git-p4.allowSubmit")
1932 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1933 die("%s is not in git-p4.allowSubmit" % self
.master
)
1935 [upstream
, settings
] = findUpstreamBranchPoint()
1936 self
.depotPath
= settings
['depot-paths'][0]
1937 if len(self
.origin
) == 0:
1938 self
.origin
= upstream
1940 if self
.update_shelve
:
1943 if self
.preserveUser
:
1944 if not self
.canChangeChangelists():
1945 die("Cannot preserve user names without p4 super-user or admin permissions")
1947 # if not set from the command line, try the config file
1948 if self
.conflict_behavior
is None:
1949 val
= gitConfig("git-p4.conflict")
1951 if val
not in self
.conflict_behavior_choices
:
1952 die("Invalid value '%s' for config git-p4.conflict" % val
)
1955 self
.conflict_behavior
= val
1958 print "Origin branch is " + self
.origin
1960 if len(self
.depotPath
) == 0:
1961 print "Internal error: cannot locate perforce depot path from existing branches"
1964 self
.useClientSpec
= False
1965 if gitConfigBool("git-p4.useclientspec"):
1966 self
.useClientSpec
= True
1967 if self
.useClientSpec
:
1968 self
.clientSpecDirs
= getClientSpec()
1970 # Check for the existence of P4 branches
1971 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
1973 if self
.useClientSpec
and not branchesDetected
:
1974 # all files are relative to the client spec
1975 self
.clientPath
= getClientRoot()
1977 self
.clientPath
= p4Where(self
.depotPath
)
1979 if self
.clientPath
== "":
1980 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1982 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1983 self
.oldWorkingDirectory
= os
.getcwd()
1985 # ensure the clientPath exists
1986 new_client_dir
= False
1987 if not os
.path
.exists(self
.clientPath
):
1988 new_client_dir
= True
1989 os
.makedirs(self
.clientPath
)
1991 chdir(self
.clientPath
, is_client_path
=True)
1993 print "Would synchronize p4 checkout in %s" % self
.clientPath
1995 print "Synchronizing p4 checkout..."
1997 # old one was destroyed, and maybe nobody told p4
1998 p4_sync("...", "-f")
2005 commitish
= self
.master
2009 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, commitish
)]):
2010 commits
.append(line
.strip())
2013 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2014 self
.checkAuthorship
= False
2016 self
.checkAuthorship
= True
2018 if self
.preserveUser
:
2019 self
.checkValidP4Users(commits
)
2022 # Build up a set of options to be passed to diff when
2023 # submitting each commit to p4.
2025 if self
.detectRenames
:
2026 # command-line -M arg
2027 self
.diffOpts
= "-M"
2029 # If not explicitly set check the config variable
2030 detectRenames
= gitConfig("git-p4.detectRenames")
2032 if detectRenames
.lower() == "false" or detectRenames
== "":
2034 elif detectRenames
.lower() == "true":
2035 self
.diffOpts
= "-M"
2037 self
.diffOpts
= "-M%s" % detectRenames
2039 # no command-line arg for -C or --find-copies-harder, just
2041 detectCopies
= gitConfig("git-p4.detectCopies")
2042 if detectCopies
.lower() == "false" or detectCopies
== "":
2044 elif detectCopies
.lower() == "true":
2045 self
.diffOpts
+= " -C"
2047 self
.diffOpts
+= " -C%s" % detectCopies
2049 if gitConfigBool("git-p4.detectCopiesHarder"):
2050 self
.diffOpts
+= " --find-copies-harder"
2053 # Apply the commits, one at a time. On failure, ask if should
2054 # continue to try the rest of the patches, or quit.
2059 last
= len(commits
) - 1
2060 for i
, commit
in enumerate(commits
):
2062 print " ", read_pipe(["git", "show", "-s",
2063 "--format=format:%h %s", commit
])
2066 ok
= self
.applyCommit(commit
)
2068 applied
.append(commit
)
2070 if self
.prepare_p4_only
and i
< last
:
2071 print "Processing only the first commit due to option" \
2072 " --prepare-p4-only"
2077 # prompt for what to do, or use the option/variable
2078 if self
.conflict_behavior
== "ask":
2079 print "What do you want to do?"
2080 response
= raw_input("[s]kip this commit but apply"
2081 " the rest, or [q]uit? ")
2084 elif self
.conflict_behavior
== "skip":
2086 elif self
.conflict_behavior
== "quit":
2089 die("Unknown conflict_behavior '%s'" %
2090 self
.conflict_behavior
)
2092 if response
[0] == "s":
2093 print "Skipping this commit, but applying the rest"
2095 if response
[0] == "q":
2102 chdir(self
.oldWorkingDirectory
)
2103 shelved_applied
= "shelved" if self
.shelve
else "applied"
2106 elif self
.prepare_p4_only
:
2108 elif len(commits
) == len(applied
):
2109 print ("All commits {0}!".format(shelved_applied
))
2113 sync
.branch
= self
.branch
2120 if len(applied
) == 0:
2121 print ("No commits {0}.".format(shelved_applied
))
2123 print ("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2129 print star
, read_pipe(["git", "show", "-s",
2130 "--format=format:%h %s", c
])
2131 print "You will have to do 'git p4 sync' and rebase."
2133 if gitConfigBool("git-p4.exportLabels"):
2134 self
.exportLabels
= True
2136 if self
.exportLabels
:
2137 p4Labels
= getP4Labels(self
.depotPath
)
2138 gitTags
= getGitTags()
2140 missingGitTags
= gitTags
- p4Labels
2141 self
.exportGitTags(missingGitTags
)
2143 # exit with error unless everything applied perfectly
2144 if len(commits
) != len(applied
):
2150 """Represent a p4 view ("p4 help views"), and map files in a
2151 repo according to the view."""
2153 def __init__(self
, client_name
):
2155 self
.client_prefix
= "//%s/" % client_name
2156 # cache results of "p4 where" to lookup client file locations
2157 self
.client_spec_path_cache
= {}
2159 def append(self
, view_line
):
2160 """Parse a view line, splitting it into depot and client
2161 sides. Append to self.mappings, preserving order. This
2162 is only needed for tag creation."""
2164 # Split the view line into exactly two words. P4 enforces
2165 # structure on these lines that simplifies this quite a bit.
2167 # Either or both words may be double-quoted.
2168 # Single quotes do not matter.
2169 # Double-quote marks cannot occur inside the words.
2170 # A + or - prefix is also inside the quotes.
2171 # There are no quotes unless they contain a space.
2172 # The line is already white-space stripped.
2173 # The two words are separated by a single space.
2175 if view_line
[0] == '"':
2176 # First word is double quoted. Find its end.
2177 close_quote_index
= view_line
.find('"', 1)
2178 if close_quote_index
<= 0:
2179 die("No first-word closing quote found: %s" % view_line
)
2180 depot_side
= view_line
[1:close_quote_index
]
2181 # skip closing quote and space
2182 rhs_index
= close_quote_index
+ 1 + 1
2184 space_index
= view_line
.find(" ")
2185 if space_index
<= 0:
2186 die("No word-splitting space found: %s" % view_line
)
2187 depot_side
= view_line
[0:space_index
]
2188 rhs_index
= space_index
+ 1
2190 # prefix + means overlay on previous mapping
2191 if depot_side
.startswith("+"):
2192 depot_side
= depot_side
[1:]
2194 # prefix - means exclude this path, leave out of mappings
2196 if depot_side
.startswith("-"):
2198 depot_side
= depot_side
[1:]
2201 self
.mappings
.append(depot_side
)
2203 def convert_client_path(self
, clientFile
):
2204 # chop off //client/ part to make it relative
2205 if not clientFile
.startswith(self
.client_prefix
):
2206 die("No prefix '%s' on clientFile '%s'" %
2207 (self
.client_prefix
, clientFile
))
2208 return clientFile
[len(self
.client_prefix
):]
2210 def update_client_spec_path_cache(self
, files
):
2211 """ Caching file paths by "p4 where" batch query """
2213 # List depot file paths exclude that already cached
2214 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2216 if len(fileArgs
) == 0:
2217 return # All files in cache
2219 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2220 for res
in where_result
:
2221 if "code" in res
and res
["code"] == "error":
2222 # assume error is "... file(s) not in client view"
2224 if "clientFile" not in res
:
2225 die("No clientFile in 'p4 where' output")
2227 # it will list all of them, but only one not unmap-ped
2229 if gitConfigBool("core.ignorecase"):
2230 res
['depotFile'] = res
['depotFile'].lower()
2231 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2233 # not found files or unmap files set to ""
2234 for depotFile
in fileArgs
:
2235 if gitConfigBool("core.ignorecase"):
2236 depotFile
= depotFile
.lower()
2237 if depotFile
not in self
.client_spec_path_cache
:
2238 self
.client_spec_path_cache
[depotFile
] = ""
2240 def map_in_client(self
, depot_path
):
2241 """Return the relative location in the client where this
2242 depot file should live. Returns "" if the file should
2243 not be mapped in the client."""
2245 if gitConfigBool("core.ignorecase"):
2246 depot_path
= depot_path
.lower()
2248 if depot_path
in self
.client_spec_path_cache
:
2249 return self
.client_spec_path_cache
[depot_path
]
2251 die( "Error: %s is not found in client spec path" % depot_path
)
2254 class P4Sync(Command
, P4UserMap
):
2255 delete_actions
= ( "delete", "move/delete", "purge" )
2258 Command
.__init
__(self
)
2259 P4UserMap
.__init
__(self
)
2261 optparse
.make_option("--branch", dest
="branch"),
2262 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2263 optparse
.make_option("--changesfile", dest
="changesFile"),
2264 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2265 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2266 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2267 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2268 help="Import into refs/heads/ , not refs/remotes"),
2269 optparse
.make_option("--max-changes", dest
="maxChanges",
2270 help="Maximum number of changes to import"),
2271 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2272 help="Internal block size to use when iteratively calling p4 changes"),
2273 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2274 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2275 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2276 help="Only sync files that are included in the Perforce Client Spec"),
2277 optparse
.make_option("-/", dest
="cloneExclude",
2278 action
="append", type="string",
2279 help="exclude depot path"),
2281 self
.description
= """Imports from Perforce into a git repository.\n
2283 //depot/my/project/ -- to import the current head
2284 //depot/my/project/@all -- to import everything
2285 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2287 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2289 self
.usage
+= " //depot/path[@revRange]"
2291 self
.createdBranches
= set()
2292 self
.committedChanges
= set()
2294 self
.detectBranches
= False
2295 self
.detectLabels
= False
2296 self
.importLabels
= False
2297 self
.changesFile
= ""
2298 self
.syncWithOrigin
= True
2299 self
.importIntoRemotes
= True
2300 self
.maxChanges
= ""
2301 self
.changes_block_size
= None
2302 self
.keepRepoPath
= False
2303 self
.depotPaths
= None
2304 self
.p4BranchesInGit
= []
2305 self
.cloneExclude
= []
2306 self
.useClientSpec
= False
2307 self
.useClientSpec_from_options
= False
2308 self
.clientSpecDirs
= None
2309 self
.tempBranches
= []
2310 self
.tempBranchLocation
= "refs/git-p4-tmp"
2311 self
.largeFileSystem
= None
2313 if gitConfig('git-p4.largeFileSystem'):
2314 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2315 self
.largeFileSystem
= largeFileSystemConstructor(
2316 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2319 if gitConfig("git-p4.syncFromOrigin") == "false":
2320 self
.syncWithOrigin
= False
2322 # This is required for the "append" cloneExclude action
2323 def ensure_value(self
, attr
, value
):
2324 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2325 setattr(self
, attr
, value
)
2326 return getattr(self
, attr
)
2328 # Force a checkpoint in fast-import and wait for it to finish
2329 def checkpoint(self
):
2330 self
.gitStream
.write("checkpoint\n\n")
2331 self
.gitStream
.write("progress checkpoint\n\n")
2332 out
= self
.gitOutput
.readline()
2334 print "checkpoint finished: " + out
2336 def extractFilesFromCommit(self
, commit
):
2337 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2338 for path
in self
.cloneExclude
]
2341 while commit
.has_key("depotFile%s" % fnum
):
2342 path
= commit
["depotFile%s" % fnum
]
2344 if [p
for p
in self
.cloneExclude
2345 if p4PathStartsWith(path
, p
)]:
2348 found
= [p
for p
in self
.depotPaths
2349 if p4PathStartsWith(path
, p
)]
2356 file["rev"] = commit
["rev%s" % fnum
]
2357 file["action"] = commit
["action%s" % fnum
]
2358 file["type"] = commit
["type%s" % fnum
]
2363 def extractJobsFromCommit(self
, commit
):
2366 while commit
.has_key("job%s" % jnum
):
2367 job
= commit
["job%s" % jnum
]
2372 def stripRepoPath(self
, path
, prefixes
):
2373 """When streaming files, this is called to map a p4 depot path
2374 to where it should go in git. The prefixes are either
2375 self.depotPaths, or self.branchPrefixes in the case of
2376 branch detection."""
2378 if self
.useClientSpec
:
2379 # branch detection moves files up a level (the branch name)
2380 # from what client spec interpretation gives
2381 path
= self
.clientSpecDirs
.map_in_client(path
)
2382 if self
.detectBranches
:
2383 for b
in self
.knownBranches
:
2384 if path
.startswith(b
+ "/"):
2385 path
= path
[len(b
)+1:]
2387 elif self
.keepRepoPath
:
2388 # Preserve everything in relative path name except leading
2389 # //depot/; just look at first prefix as they all should
2390 # be in the same depot.
2391 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2392 if p4PathStartsWith(path
, depot
):
2393 path
= path
[len(depot
):]
2397 if p4PathStartsWith(path
, p
):
2398 path
= path
[len(p
):]
2401 path
= wildcard_decode(path
)
2404 def splitFilesIntoBranches(self
, commit
):
2405 """Look at each depotFile in the commit to figure out to what
2406 branch it belongs."""
2408 if self
.clientSpecDirs
:
2409 files
= self
.extractFilesFromCommit(commit
)
2410 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2414 while commit
.has_key("depotFile%s" % fnum
):
2415 path
= commit
["depotFile%s" % fnum
]
2416 found
= [p
for p
in self
.depotPaths
2417 if p4PathStartsWith(path
, p
)]
2424 file["rev"] = commit
["rev%s" % fnum
]
2425 file["action"] = commit
["action%s" % fnum
]
2426 file["type"] = commit
["type%s" % fnum
]
2429 # start with the full relative path where this file would
2431 if self
.useClientSpec
:
2432 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2434 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2436 for branch
in self
.knownBranches
.keys():
2437 # add a trailing slash so that a commit into qt/4.2foo
2438 # doesn't end up in qt/4.2, e.g.
2439 if relPath
.startswith(branch
+ "/"):
2440 if branch
not in branches
:
2441 branches
[branch
] = []
2442 branches
[branch
].append(file)
2447 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2448 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2449 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2451 self
.gitStream
.write(d
)
2452 self
.gitStream
.write('\n')
2454 # output one file from the P4 stream
2455 # - helper for streamP4Files
2457 def streamOneP4File(self
, file, contents
):
2458 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2460 size
= int(self
.stream_file
['fileSize'])
2461 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2464 (type_base
, type_mods
) = split_p4_type(file["type"])
2467 if "x" in type_mods
:
2469 if type_base
== "symlink":
2471 # p4 print on a symlink sometimes contains "target\n";
2472 # if it does, remove the newline
2473 data
= ''.join(contents
)
2475 # Some version of p4 allowed creating a symlink that pointed
2476 # to nothing. This causes p4 errors when checking out such
2477 # a change, and errors here too. Work around it by ignoring
2478 # the bad symlink; hopefully a future change fixes it.
2479 print "\nIgnoring empty symlink in %s" % file['depotFile']
2481 elif data
[-1] == '\n':
2482 contents
= [data
[:-1]]
2486 if type_base
== "utf16":
2487 # p4 delivers different text in the python output to -G
2488 # than it does when using "print -o", or normal p4 client
2489 # operations. utf16 is converted to ascii or utf8, perhaps.
2490 # But ascii text saved as -t utf16 is completely mangled.
2491 # Invoke print -o to get the real contents.
2493 # On windows, the newlines will always be mangled by print, so put
2494 # them back too. This is not needed to the cygwin windows version,
2495 # just the native "NT" type.
2498 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2499 except Exception as e
:
2500 if 'Translation of file content failed' in str(e
):
2501 type_base
= 'binary'
2505 if p4_version_string().find('/NT') >= 0:
2506 text
= text
.replace('\r\n', '\n')
2509 if type_base
== "apple":
2510 # Apple filetype files will be streamed as a concatenation of
2511 # its appledouble header and the contents. This is useless
2512 # on both macs and non-macs. If using "print -q -o xx", it
2513 # will create "xx" with the data, and "%xx" with the header.
2514 # This is also not very useful.
2516 # Ideally, someday, this script can learn how to generate
2517 # appledouble files directly and import those to git, but
2518 # non-mac machines can never find a use for apple filetype.
2519 print "\nIgnoring apple filetype file %s" % file['depotFile']
2522 # Note that we do not try to de-mangle keywords on utf16 files,
2523 # even though in theory somebody may want that.
2524 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2526 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2527 text
= ''.join(contents
)
2528 text
= regexp
.sub(r
'$\1$', text
)
2532 relPath
.decode('ascii')
2535 if gitConfig('git-p4.pathEncoding'):
2536 encoding
= gitConfig('git-p4.pathEncoding')
2537 relPath
= relPath
.decode(encoding
, 'replace').encode('utf8', 'replace')
2539 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, relPath
)
2541 if self
.largeFileSystem
:
2542 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2544 self
.writeToGitStream(git_mode
, relPath
, contents
)
2546 def streamOneP4Deletion(self
, file):
2547 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2549 sys
.stdout
.write("delete %s\n" % relPath
)
2551 self
.gitStream
.write("D %s\n" % relPath
)
2553 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2554 self
.largeFileSystem
.removeLargeFile(relPath
)
2556 # handle another chunk of streaming data
2557 def streamP4FilesCb(self
, marshalled
):
2559 # catch p4 errors and complain
2561 if "code" in marshalled
:
2562 if marshalled
["code"] == "error":
2563 if "data" in marshalled
:
2564 err
= marshalled
["data"].rstrip()
2566 if not err
and 'fileSize' in self
.stream_file
:
2567 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2568 if required_bytes
> 0:
2569 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2570 os
.getcwd(), required_bytes
/1024/1024
2575 if self
.stream_have_file_info
:
2576 if "depotFile" in self
.stream_file
:
2577 f
= self
.stream_file
["depotFile"]
2578 # force a failure in fast-import, else an empty
2579 # commit will be made
2580 self
.gitStream
.write("\n")
2581 self
.gitStream
.write("die-now\n")
2582 self
.gitStream
.close()
2583 # ignore errors, but make sure it exits first
2584 self
.importProcess
.wait()
2586 die("Error from p4 print for %s: %s" % (f
, err
))
2588 die("Error from p4 print: %s" % err
)
2590 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2591 # start of a new file - output the old one first
2592 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2593 self
.stream_file
= {}
2594 self
.stream_contents
= []
2595 self
.stream_have_file_info
= False
2597 # pick up the new file information... for the
2598 # 'data' field we need to append to our array
2599 for k
in marshalled
.keys():
2601 if 'streamContentSize' not in self
.stream_file
:
2602 self
.stream_file
['streamContentSize'] = 0
2603 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2604 self
.stream_contents
.append(marshalled
['data'])
2606 self
.stream_file
[k
] = marshalled
[k
]
2609 'streamContentSize' in self
.stream_file
and
2610 'fileSize' in self
.stream_file
and
2611 'depotFile' in self
.stream_file
):
2612 size
= int(self
.stream_file
["fileSize"])
2614 progress
= 100*self
.stream_file
['streamContentSize']/size
2615 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2618 self
.stream_have_file_info
= True
2620 # Stream directly from "p4 files" into "git fast-import"
2621 def streamP4Files(self
, files
):
2627 filesForCommit
.append(f
)
2628 if f
['action'] in self
.delete_actions
:
2629 filesToDelete
.append(f
)
2631 filesToRead
.append(f
)
2634 for f
in filesToDelete
:
2635 self
.streamOneP4Deletion(f
)
2637 if len(filesToRead
) > 0:
2638 self
.stream_file
= {}
2639 self
.stream_contents
= []
2640 self
.stream_have_file_info
= False
2642 # curry self argument
2643 def streamP4FilesCbSelf(entry
):
2644 self
.streamP4FilesCb(entry
)
2646 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2648 p4CmdList(["-x", "-", "print"],
2650 cb
=streamP4FilesCbSelf
)
2653 if self
.stream_file
.has_key('depotFile'):
2654 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2656 def make_email(self
, userid
):
2657 if userid
in self
.users
:
2658 return self
.users
[userid
]
2660 return "%s <a@b>" % userid
2662 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2663 """ Stream a p4 tag.
2664 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2668 print "writing tag %s for commit %s" % (labelName
, commit
)
2669 gitStream
.write("tag %s\n" % labelName
)
2670 gitStream
.write("from %s\n" % commit
)
2672 if labelDetails
.has_key('Owner'):
2673 owner
= labelDetails
["Owner"]
2677 # Try to use the owner of the p4 label, or failing that,
2678 # the current p4 user id.
2680 email
= self
.make_email(owner
)
2682 email
= self
.make_email(self
.p4UserId())
2683 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2685 gitStream
.write("tagger %s\n" % tagger
)
2687 print "labelDetails=",labelDetails
2688 if labelDetails
.has_key('Description'):
2689 description
= labelDetails
['Description']
2691 description
= 'Label from git p4'
2693 gitStream
.write("data %d\n" % len(description
))
2694 gitStream
.write(description
)
2695 gitStream
.write("\n")
2697 def inClientSpec(self
, path
):
2698 if not self
.clientSpecDirs
:
2700 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
2701 if not inClientSpec
and self
.verbose
:
2702 print('Ignoring file outside of client spec: {0}'.format(path
))
2705 def hasBranchPrefix(self
, path
):
2706 if not self
.branchPrefixes
:
2708 hasPrefix
= [p
for p
in self
.branchPrefixes
2709 if p4PathStartsWith(path
, p
)]
2710 if not hasPrefix
and self
.verbose
:
2711 print('Ignoring file outside of prefix: {0}'.format(path
))
2714 def commit(self
, details
, files
, branch
, parent
= ""):
2715 epoch
= details
["time"]
2716 author
= details
["user"]
2717 jobs
= self
.extractJobsFromCommit(details
)
2720 print('commit into {0}'.format(branch
))
2722 if self
.clientSpecDirs
:
2723 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2725 files
= [f
for f
in files
2726 if self
.inClientSpec(f
['path']) and self
.hasBranchPrefix(f
['path'])]
2728 if not files
and not gitConfigBool('git-p4.keepEmptyCommits'):
2729 print('Ignoring revision {0} as it would produce an empty commit.'
2730 .format(details
['change']))
2733 self
.gitStream
.write("commit %s\n" % branch
)
2734 self
.gitStream
.write("mark :%s\n" % details
["change"])
2735 self
.committedChanges
.add(int(details
["change"]))
2737 if author
not in self
.users
:
2738 self
.getUserMapFromPerforceServer()
2739 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2741 self
.gitStream
.write("committer %s\n" % committer
)
2743 self
.gitStream
.write("data <<EOT\n")
2744 self
.gitStream
.write(details
["desc"])
2746 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
2747 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2748 (','.join(self
.branchPrefixes
), details
["change"]))
2749 if len(details
['options']) > 0:
2750 self
.gitStream
.write(": options = %s" % details
['options'])
2751 self
.gitStream
.write("]\nEOT\n\n")
2755 print "parent %s" % parent
2756 self
.gitStream
.write("from %s\n" % parent
)
2758 self
.streamP4Files(files
)
2759 self
.gitStream
.write("\n")
2761 change
= int(details
["change"])
2763 if self
.labels
.has_key(change
):
2764 label
= self
.labels
[change
]
2765 labelDetails
= label
[0]
2766 labelRevisions
= label
[1]
2768 print "Change %s is labelled %s" % (change
, labelDetails
)
2770 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2771 for p
in self
.branchPrefixes
])
2773 if len(files
) == len(labelRevisions
):
2777 if info
["action"] in self
.delete_actions
:
2779 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2781 if cleanedFiles
== labelRevisions
:
2782 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2786 print ("Tag %s does not match with change %s: files do not match."
2787 % (labelDetails
["label"], change
))
2791 print ("Tag %s does not match with change %s: file count is different."
2792 % (labelDetails
["label"], change
))
2794 # Build a dictionary of changelists and labels, for "detect-labels" option.
2795 def getLabels(self
):
2798 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2799 if len(l
) > 0 and not self
.silent
:
2800 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2803 label
= output
["label"]
2807 print "Querying files for label %s" % label
2808 for file in p4CmdList(["files"] +
2809 ["%s...@%s" % (p
, label
)
2810 for p
in self
.depotPaths
]):
2811 revisions
[file["depotFile"]] = file["rev"]
2812 change
= int(file["change"])
2813 if change
> newestChange
:
2814 newestChange
= change
2816 self
.labels
[newestChange
] = [output
, revisions
]
2819 print "Label changes: %s" % self
.labels
.keys()
2821 # Import p4 labels as git tags. A direct mapping does not
2822 # exist, so assume that if all the files are at the same revision
2823 # then we can use that, or it's something more complicated we should
2825 def importP4Labels(self
, stream
, p4Labels
):
2827 print "import p4 labels: " + ' '.join(p4Labels
)
2829 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2830 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2831 if len(validLabelRegexp
) == 0:
2832 validLabelRegexp
= defaultLabelRegexp
2833 m
= re
.compile(validLabelRegexp
)
2835 for name
in p4Labels
:
2838 if not m
.match(name
):
2840 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2843 if name
in ignoredP4Labels
:
2846 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2848 # get the most recent changelist for each file in this label
2849 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2850 for p
in self
.depotPaths
])
2852 if change
.has_key('change'):
2853 # find the corresponding git commit; take the oldest commit
2854 changelist
= int(change
['change'])
2855 if changelist
in self
.committedChanges
:
2856 gitCommit
= ":%d" % changelist
# use a fast-import mark
2859 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2860 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
2861 if len(gitCommit
) == 0:
2862 print "importing label %s: could not find git commit for changelist %d" % (name
, changelist
)
2865 gitCommit
= gitCommit
.strip()
2868 # Convert from p4 time format
2870 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2872 print "Could not convert label time %s" % labelDetails
['Update']
2875 when
= int(time
.mktime(tmwhen
))
2876 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2878 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2881 print "Label %s has no changelists - possibly deleted?" % name
2884 # We can't import this label; don't try again as it will get very
2885 # expensive repeatedly fetching all the files for labels that will
2886 # never be imported. If the label is moved in the future, the
2887 # ignore will need to be removed manually.
2888 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2890 def guessProjectName(self
):
2891 for p
in self
.depotPaths
:
2894 p
= p
[p
.strip().rfind("/") + 1:]
2895 if not p
.endswith("/"):
2899 def getBranchMapping(self
):
2900 lostAndFoundBranches
= set()
2902 user
= gitConfig("git-p4.branchUser")
2904 command
= "branches -u %s" % user
2906 command
= "branches"
2908 for info
in p4CmdList(command
):
2909 details
= p4Cmd(["branch", "-o", info
["branch"]])
2911 while details
.has_key("View%s" % viewIdx
):
2912 paths
= details
["View%s" % viewIdx
].split(" ")
2913 viewIdx
= viewIdx
+ 1
2914 # require standard //depot/foo/... //depot/bar/... mapping
2915 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2918 destination
= paths
[1]
2920 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2921 source
= source
[len(self
.depotPaths
[0]):-4]
2922 destination
= destination
[len(self
.depotPaths
[0]):-4]
2924 if destination
in self
.knownBranches
:
2926 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2927 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2930 self
.knownBranches
[destination
] = source
2932 lostAndFoundBranches
.discard(destination
)
2934 if source
not in self
.knownBranches
:
2935 lostAndFoundBranches
.add(source
)
2937 # Perforce does not strictly require branches to be defined, so we also
2938 # check git config for a branch list.
2940 # Example of branch definition in git config file:
2942 # branchList=main:branchA
2943 # branchList=main:branchB
2944 # branchList=branchA:branchC
2945 configBranches
= gitConfigList("git-p4.branchList")
2946 for branch
in configBranches
:
2948 (source
, destination
) = branch
.split(":")
2949 self
.knownBranches
[destination
] = source
2951 lostAndFoundBranches
.discard(destination
)
2953 if source
not in self
.knownBranches
:
2954 lostAndFoundBranches
.add(source
)
2957 for branch
in lostAndFoundBranches
:
2958 self
.knownBranches
[branch
] = branch
2960 def getBranchMappingFromGitBranches(self
):
2961 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2962 for branch
in branches
.keys():
2963 if branch
== "master":
2966 branch
= branch
[len(self
.projectName
):]
2967 self
.knownBranches
[branch
] = branch
2969 def updateOptionDict(self
, d
):
2971 if self
.keepRepoPath
:
2972 option_keys
['keepRepoPath'] = 1
2974 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2976 def readOptions(self
, d
):
2977 self
.keepRepoPath
= (d
.has_key('options')
2978 and ('keepRepoPath' in d
['options']))
2980 def gitRefForBranch(self
, branch
):
2981 if branch
== "main":
2982 return self
.refPrefix
+ "master"
2984 if len(branch
) <= 0:
2987 return self
.refPrefix
+ self
.projectName
+ branch
2989 def gitCommitByP4Change(self
, ref
, change
):
2991 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2994 latestCommit
= parseRevision(ref
)
2998 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2999 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
3004 log
= extractLogMessageFromGitCommit(next
)
3005 settings
= extractSettingsGitLog(log
)
3006 currentChange
= int(settings
['change'])
3008 print "current change %s" % currentChange
3010 if currentChange
== change
:
3012 print "found %s" % next
3015 if currentChange
< change
:
3016 earliestCommit
= "^%s" % next
3018 latestCommit
= "%s" % next
3022 def importNewBranch(self
, branch
, maxChange
):
3023 # make fast-import flush all changes to disk and update the refs using the checkpoint
3024 # command so that we can try to find the branch parent in the git history
3025 self
.gitStream
.write("checkpoint\n\n");
3026 self
.gitStream
.flush();
3027 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3028 range = "@1,%s" % maxChange
3029 #print "prefix" + branchPrefix
3030 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3031 if len(changes
) <= 0:
3033 firstChange
= changes
[0]
3034 #print "first change in branch: %s" % firstChange
3035 sourceBranch
= self
.knownBranches
[branch
]
3036 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3037 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3038 #print "source " + sourceBranch
3040 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3041 #print "branch parent: %s" % branchParentChange
3042 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3043 if len(gitParent
) > 0:
3044 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3045 #print "parent git commit: %s" % gitParent
3047 self
.importChanges(changes
)
3050 def searchParent(self
, parent
, branch
, target
):
3052 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
3053 "--no-merges", parent
]):
3055 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
3058 print "Found parent of %s in commit %s" % (branch
, blob
)
3065 def importChanges(self
, changes
):
3067 for change
in changes
:
3068 description
= p4_describe(change
)
3069 self
.updateOptionDict(description
)
3072 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3077 if self
.detectBranches
:
3078 branches
= self
.splitFilesIntoBranches(description
)
3079 for branch
in branches
.keys():
3081 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3082 self
.branchPrefixes
= [ branchPrefix
]
3086 filesForCommit
= branches
[branch
]
3089 print "branch is %s" % branch
3091 self
.updatedBranches
.add(branch
)
3093 if branch
not in self
.createdBranches
:
3094 self
.createdBranches
.add(branch
)
3095 parent
= self
.knownBranches
[branch
]
3096 if parent
== branch
:
3099 fullBranch
= self
.projectName
+ branch
3100 if fullBranch
not in self
.p4BranchesInGit
:
3102 print("\n Importing new branch %s" % fullBranch
);
3103 if self
.importNewBranch(branch
, change
- 1):
3105 self
.p4BranchesInGit
.append(fullBranch
)
3107 print("\n Resuming with change %s" % change
);
3110 print "parent determined through known branches: %s" % parent
3112 branch
= self
.gitRefForBranch(branch
)
3113 parent
= self
.gitRefForBranch(parent
)
3116 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
3118 if len(parent
) == 0 and branch
in self
.initialParents
:
3119 parent
= self
.initialParents
[branch
]
3120 del self
.initialParents
[branch
]
3124 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3126 print "Creating temporary branch: " + tempBranch
3127 self
.commit(description
, filesForCommit
, tempBranch
)
3128 self
.tempBranches
.append(tempBranch
)
3130 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3132 self
.commit(description
, filesForCommit
, branch
, blob
)
3135 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3136 self
.commit(description
, filesForCommit
, branch
, parent
)
3138 files
= self
.extractFilesFromCommit(description
)
3139 self
.commit(description
, files
, self
.branch
,
3141 # only needed once, to connect to the previous commit
3142 self
.initialParent
= ""
3144 print self
.gitError
.read()
3147 def importHeadRevision(self
, revision
):
3148 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3151 details
["user"] = "git perforce import user"
3152 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3153 % (' '.join(self
.depotPaths
), revision
))
3154 details
["change"] = revision
3158 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3160 for info
in p4CmdList(["files"] + fileArgs
):
3162 if 'code' in info
and info
['code'] == 'error':
3163 sys
.stderr
.write("p4 returned an error: %s\n"
3165 if info
['data'].find("must refer to client") >= 0:
3166 sys
.stderr
.write("This particular p4 error is misleading.\n")
3167 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3168 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3170 if 'p4ExitCode' in info
:
3171 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3175 change
= int(info
["change"])
3176 if change
> newestRevision
:
3177 newestRevision
= change
3179 if info
["action"] in self
.delete_actions
:
3180 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3181 #fileCnt = fileCnt + 1
3184 for prop
in ["depotFile", "rev", "action", "type" ]:
3185 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3187 fileCnt
= fileCnt
+ 1
3189 details
["change"] = newestRevision
3191 # Use time from top-most change so that all git p4 clones of
3192 # the same p4 repo have the same commit SHA1s.
3193 res
= p4_describe(newestRevision
)
3194 details
["time"] = res
["time"]
3196 self
.updateOptionDict(details
)
3198 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3200 print "IO error with git fast-import. Is your git version recent enough?"
3201 print self
.gitError
.read()
3204 def run(self
, args
):
3205 self
.depotPaths
= []
3206 self
.changeRange
= ""
3207 self
.previousDepotPaths
= []
3208 self
.hasOrigin
= False
3210 # map from branch depot path to parent branch
3211 self
.knownBranches
= {}
3212 self
.initialParents
= {}
3214 if self
.importIntoRemotes
:
3215 self
.refPrefix
= "refs/remotes/p4/"
3217 self
.refPrefix
= "refs/heads/p4/"
3219 if self
.syncWithOrigin
:
3220 self
.hasOrigin
= originP4BranchesExist()
3223 print 'Syncing with origin first, using "git fetch origin"'
3224 system("git fetch origin")
3226 branch_arg_given
= bool(self
.branch
)
3227 if len(self
.branch
) == 0:
3228 self
.branch
= self
.refPrefix
+ "master"
3229 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3230 system("git update-ref %s refs/heads/p4" % self
.branch
)
3231 system("git branch -D p4")
3233 # accept either the command-line option, or the configuration variable
3234 if self
.useClientSpec
:
3235 # will use this after clone to set the variable
3236 self
.useClientSpec_from_options
= True
3238 if gitConfigBool("git-p4.useclientspec"):
3239 self
.useClientSpec
= True
3240 if self
.useClientSpec
:
3241 self
.clientSpecDirs
= getClientSpec()
3243 # TODO: should always look at previous commits,
3244 # merge with previous imports, if possible.
3247 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3249 # branches holds mapping from branch name to sha1
3250 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3252 # restrict to just this one, disabling detect-branches
3253 if branch_arg_given
:
3254 short
= self
.branch
.split("/")[-1]
3255 if short
in branches
:
3256 self
.p4BranchesInGit
= [ short
]
3258 self
.p4BranchesInGit
= branches
.keys()
3260 if len(self
.p4BranchesInGit
) > 1:
3262 print "Importing from/into multiple branches"
3263 self
.detectBranches
= True
3264 for branch
in branches
.keys():
3265 self
.initialParents
[self
.refPrefix
+ branch
] = \
3269 print "branches: %s" % self
.p4BranchesInGit
3272 for branch
in self
.p4BranchesInGit
:
3273 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3275 settings
= extractSettingsGitLog(logMsg
)
3277 self
.readOptions(settings
)
3278 if (settings
.has_key('depot-paths')
3279 and settings
.has_key ('change')):
3280 change
= int(settings
['change']) + 1
3281 p4Change
= max(p4Change
, change
)
3283 depotPaths
= sorted(settings
['depot-paths'])
3284 if self
.previousDepotPaths
== []:
3285 self
.previousDepotPaths
= depotPaths
3288 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3289 prev_list
= prev
.split("/")
3290 cur_list
= cur
.split("/")
3291 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3292 if cur_list
[i
] <> prev_list
[i
]:
3296 paths
.append ("/".join(cur_list
[:i
+ 1]))
3298 self
.previousDepotPaths
= paths
3301 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3302 self
.changeRange
= "@%s,#head" % p4Change
3303 if not self
.silent
and not self
.detectBranches
:
3304 print "Performing incremental import into %s git branch" % self
.branch
3306 # accept multiple ref name abbreviations:
3307 # refs/foo/bar/branch -> use it exactly
3308 # p4/branch -> prepend refs/remotes/ or refs/heads/
3309 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3310 if not self
.branch
.startswith("refs/"):
3311 if self
.importIntoRemotes
:
3312 prepend
= "refs/remotes/"
3314 prepend
= "refs/heads/"
3315 if not self
.branch
.startswith("p4/"):
3317 self
.branch
= prepend
+ self
.branch
3319 if len(args
) == 0 and self
.depotPaths
:
3321 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3323 if self
.depotPaths
and self
.depotPaths
!= args
:
3324 print ("previous import used depot path %s and now %s was specified. "
3325 "This doesn't work!" % (' '.join (self
.depotPaths
),
3329 self
.depotPaths
= sorted(args
)
3334 # Make sure no revision specifiers are used when --changesfile
3336 bad_changesfile
= False
3337 if len(self
.changesFile
) > 0:
3338 for p
in self
.depotPaths
:
3339 if p
.find("@") >= 0 or p
.find("#") >= 0:
3340 bad_changesfile
= True
3343 die("Option --changesfile is incompatible with revision specifiers")
3346 for p
in self
.depotPaths
:
3347 if p
.find("@") != -1:
3348 atIdx
= p
.index("@")
3349 self
.changeRange
= p
[atIdx
:]
3350 if self
.changeRange
== "@all":
3351 self
.changeRange
= ""
3352 elif ',' not in self
.changeRange
:
3353 revision
= self
.changeRange
3354 self
.changeRange
= ""
3356 elif p
.find("#") != -1:
3357 hashIdx
= p
.index("#")
3358 revision
= p
[hashIdx
:]
3360 elif self
.previousDepotPaths
== []:
3361 # pay attention to changesfile, if given, else import
3362 # the entire p4 tree at the head revision
3363 if len(self
.changesFile
) == 0:
3366 p
= re
.sub ("\.\.\.$", "", p
)
3367 if not p
.endswith("/"):
3372 self
.depotPaths
= newPaths
3374 # --detect-branches may change this for each branch
3375 self
.branchPrefixes
= self
.depotPaths
3377 self
.loadUserMapFromCache()
3379 if self
.detectLabels
:
3382 if self
.detectBranches
:
3383 ## FIXME - what's a P4 projectName ?
3384 self
.projectName
= self
.guessProjectName()
3387 self
.getBranchMappingFromGitBranches()
3389 self
.getBranchMapping()
3391 print "p4-git branches: %s" % self
.p4BranchesInGit
3392 print "initial parents: %s" % self
.initialParents
3393 for b
in self
.p4BranchesInGit
:
3397 b
= b
[len(self
.projectName
):]
3398 self
.createdBranches
.add(b
)
3400 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
3402 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3403 stdin
=subprocess
.PIPE
,
3404 stdout
=subprocess
.PIPE
,
3405 stderr
=subprocess
.PIPE
);
3406 self
.gitOutput
= self
.importProcess
.stdout
3407 self
.gitStream
= self
.importProcess
.stdin
3408 self
.gitError
= self
.importProcess
.stderr
3411 self
.importHeadRevision(revision
)
3415 if len(self
.changesFile
) > 0:
3416 output
= open(self
.changesFile
).readlines()
3419 changeSet
.add(int(line
))
3421 for change
in changeSet
:
3422 changes
.append(change
)
3426 # catch "git p4 sync" with no new branches, in a repo that
3427 # does not have any existing p4 branches
3429 if not self
.p4BranchesInGit
:
3430 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3432 # The default branch is master, unless --branch is used to
3433 # specify something else. Make sure it exists, or complain
3434 # nicely about how to use --branch.
3435 if not self
.detectBranches
:
3436 if not branch_exists(self
.branch
):
3437 if branch_arg_given
:
3438 die("Error: branch %s does not exist." % self
.branch
)
3440 die("Error: no branch %s; perhaps specify one with --branch." %
3444 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3446 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3448 if len(self
.maxChanges
) > 0:
3449 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3451 if len(changes
) == 0:
3453 print "No changes to import!"
3455 if not self
.silent
and not self
.detectBranches
:
3456 print "Import destination: %s" % self
.branch
3458 self
.updatedBranches
= set()
3460 if not self
.detectBranches
:
3462 # start a new branch
3463 self
.initialParent
= ""
3465 # build on a previous revision
3466 self
.initialParent
= parseRevision(self
.branch
)
3468 self
.importChanges(changes
)
3472 if len(self
.updatedBranches
) > 0:
3473 sys
.stdout
.write("Updated branches: ")
3474 for b
in self
.updatedBranches
:
3475 sys
.stdout
.write("%s " % b
)
3476 sys
.stdout
.write("\n")
3478 if gitConfigBool("git-p4.importLabels"):
3479 self
.importLabels
= True
3481 if self
.importLabels
:
3482 p4Labels
= getP4Labels(self
.depotPaths
)
3483 gitTags
= getGitTags()
3485 missingP4Labels
= p4Labels
- gitTags
3486 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3488 self
.gitStream
.close()
3489 if self
.importProcess
.wait() != 0:
3490 die("fast-import failed: %s" % self
.gitError
.read())
3491 self
.gitOutput
.close()
3492 self
.gitError
.close()
3494 # Cleanup temporary branches created during import
3495 if self
.tempBranches
!= []:
3496 for branch
in self
.tempBranches
:
3497 read_pipe("git update-ref -d %s" % branch
)
3498 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3500 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3501 # a convenient shortcut refname "p4".
3502 if self
.importIntoRemotes
:
3503 head_ref
= self
.refPrefix
+ "HEAD"
3504 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3505 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3509 class P4Rebase(Command
):
3511 Command
.__init
__(self
)
3513 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3515 self
.importLabels
= False
3516 self
.description
= ("Fetches the latest revision from perforce and "
3517 + "rebases the current work (branch) against it")
3519 def run(self
, args
):
3521 sync
.importLabels
= self
.importLabels
3524 return self
.rebase()
3527 if os
.system("git update-index --refresh") != 0:
3528 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3529 if len(read_pipe("git diff-index HEAD --")) > 0:
3530 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3532 [upstream
, settings
] = findUpstreamBranchPoint()
3533 if len(upstream
) == 0:
3534 die("Cannot find upstream branchpoint for rebase")
3536 # the branchpoint may be p4/foo~3, so strip off the parent
3537 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3539 print "Rebasing the current branch onto %s" % upstream
3540 oldHead
= read_pipe("git rev-parse HEAD").strip()
3541 system("git rebase %s" % upstream
)
3542 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3545 class P4Clone(P4Sync
):
3547 P4Sync
.__init
__(self
)
3548 self
.description
= "Creates a new git repository and imports from Perforce into it"
3549 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3551 optparse
.make_option("--destination", dest
="cloneDestination",
3552 action
='store', default
=None,
3553 help="where to leave result of the clone"),
3554 optparse
.make_option("--bare", dest
="cloneBare",
3555 action
="store_true", default
=False),
3557 self
.cloneDestination
= None
3558 self
.needsGit
= False
3559 self
.cloneBare
= False
3561 def defaultDestination(self
, args
):
3562 ## TODO: use common prefix of args?
3564 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3565 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3566 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3567 depotDir
= re
.sub(r
"/$", "", depotDir
)
3568 return os
.path
.split(depotDir
)[1]
3570 def run(self
, args
):
3574 if self
.keepRepoPath
and not self
.cloneDestination
:
3575 sys
.stderr
.write("Must specify destination for --keep-path\n")
3580 if not self
.cloneDestination
and len(depotPaths
) > 1:
3581 self
.cloneDestination
= depotPaths
[-1]
3582 depotPaths
= depotPaths
[:-1]
3584 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3585 for p
in depotPaths
:
3586 if not p
.startswith("//"):
3587 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3590 if not self
.cloneDestination
:
3591 self
.cloneDestination
= self
.defaultDestination(args
)
3593 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3595 if not os
.path
.exists(self
.cloneDestination
):
3596 os
.makedirs(self
.cloneDestination
)
3597 chdir(self
.cloneDestination
)
3599 init_cmd
= [ "git", "init" ]
3601 init_cmd
.append("--bare")
3602 retcode
= subprocess
.call(init_cmd
)
3604 raise CalledProcessError(retcode
, init_cmd
)
3606 if not P4Sync
.run(self
, depotPaths
):
3609 # create a master branch and check out a work tree
3610 if gitBranchExists(self
.branch
):
3611 system([ "git", "branch", "master", self
.branch
])
3612 if not self
.cloneBare
:
3613 system([ "git", "checkout", "-f" ])
3615 print 'Not checking out any branch, use ' \
3616 '"git checkout -q -b master <branch>"'
3618 # auto-set this variable if invoked with --use-client-spec
3619 if self
.useClientSpec_from_options
:
3620 system("git config --bool git-p4.useclientspec true")
3624 class P4Branches(Command
):
3626 Command
.__init
__(self
)
3628 self
.description
= ("Shows the git branches that hold imports and their "
3629 + "corresponding perforce depot paths")
3630 self
.verbose
= False
3632 def run(self
, args
):
3633 if originP4BranchesExist():
3634 createOrUpdateBranchesFromOrigin()
3636 cmdline
= "git rev-parse --symbolic "
3637 cmdline
+= " --remotes"
3639 for line
in read_pipe_lines(cmdline
):
3642 if not line
.startswith('p4/') or line
== "p4/HEAD":
3646 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3647 settings
= extractSettingsGitLog(log
)
3649 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3652 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3654 optparse
.IndentedHelpFormatter
.__init
__(self
)
3656 def format_description(self
, description
):
3658 return description
+ "\n"
3662 def printUsage(commands
):
3663 print "usage: %s <command> [options]" % sys
.argv
[0]
3665 print "valid commands: %s" % ", ".join(commands
)
3667 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3672 "submit" : P4Submit
,
3673 "commit" : P4Submit
,
3675 "rebase" : P4Rebase
,
3677 "rollback" : P4RollBack
,
3678 "branches" : P4Branches
3683 if len(sys
.argv
[1:]) == 0:
3684 printUsage(commands
.keys())
3687 cmdName
= sys
.argv
[1]
3689 klass
= commands
[cmdName
]
3692 print "unknown command %s" % cmdName
3694 printUsage(commands
.keys())
3697 options
= cmd
.options
3698 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3702 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3704 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3706 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3708 description
= cmd
.description
,
3709 formatter
= HelpFormatter())
3711 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3713 verbose
= cmd
.verbose
3715 if cmd
.gitdir
== None:
3716 cmd
.gitdir
= os
.path
.abspath(".git")
3717 if not isValidGitDir(cmd
.gitdir
):
3718 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3719 if os
.path
.exists(cmd
.gitdir
):
3720 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3724 if not isValidGitDir(cmd
.gitdir
):
3725 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3726 cmd
.gitdir
+= "/.git"
3728 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3730 os
.environ
["GIT_DIR"] = cmd
.gitdir
3732 if not cmd
.run(args
):
3737 if __name__
== '__main__':