3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess
import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self
, returncode
, cmd
):
39 self
.returncode
= returncode
42 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize
= 512
52 def p4_build_cmd(cmd
):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user
= gitConfig("git-p4.user")
63 real_cmd
+= ["-u",user
]
65 password
= gitConfig("git-p4.password")
67 real_cmd
+= ["-P", password
]
69 port
= gitConfig("git-p4.port")
71 real_cmd
+= ["-p", port
]
73 host
= gitConfig("git-p4.host")
75 real_cmd
+= ["-H", host
]
77 client
= gitConfig("git-p4.client")
79 real_cmd
+= ["-c", client
]
82 if isinstance(cmd
,basestring
):
83 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
88 def chdir(path
, is_client_path
=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
106 if not is_client_path
:
108 os
.environ
['PWD'] = path
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform
.system() == 'Windows':
113 free_bytes
= ctypes
.c_ulonglong(0)
114 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
115 return free_bytes
.value
117 st
= os
.statvfs(os
.getcwd())
118 return st
.f_bavail
* st
.f_frsize
124 sys
.stderr
.write(msg
+ "\n")
127 def write_pipe(c
, stdin
):
129 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
131 expand
= isinstance(c
,basestring
)
132 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
134 val
= pipe
.write(stdin
)
137 die('Command failed: %s' % str(c
))
141 def p4_write_pipe(c
, stdin
):
142 real_cmd
= p4_build_cmd(c
)
143 return write_pipe(real_cmd
, stdin
)
145 def read_pipe(c
, ignore_error
=False):
147 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
149 expand
= isinstance(c
,basestring
)
150 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
151 (out
, err
) = p
.communicate()
152 if p
.returncode
!= 0 and not ignore_error
:
153 die('Command failed: %s\nError: %s' % (str(c
), err
))
156 def p4_read_pipe(c
, ignore_error
=False):
157 real_cmd
= p4_build_cmd(c
)
158 return read_pipe(real_cmd
, ignore_error
)
160 def read_pipe_lines(c
):
162 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
164 expand
= isinstance(c
, basestring
)
165 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
167 val
= pipe
.readlines()
168 if pipe
.close() or p
.wait():
169 die('Command failed: %s' % str(c
))
173 def p4_read_pipe_lines(c
):
174 """Specifically invoke p4 on the command supplied. """
175 real_cmd
= p4_build_cmd(c
)
176 return read_pipe_lines(real_cmd
)
178 def p4_has_command(cmd
):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd
= p4_build_cmd(["help", cmd
])
182 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
183 stderr
=subprocess
.PIPE
)
185 return p
.returncode
== 0
187 def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
193 if not p4_has_command("move"):
195 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
196 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
197 (out
, err
) = p
.communicate()
198 # return code will be 1 in either case
199 if err
.find("Invalid option") >= 0:
201 if err
.find("disabled") >= 0:
203 # assume it failed because @... was invalid changelist
206 def system(cmd
, ignore_error
=False):
207 expand
= isinstance(cmd
,basestring
)
209 sys
.stderr
.write("executing %s\n" % str(cmd
))
210 retcode
= subprocess
.call(cmd
, shell
=expand
)
211 if retcode
and not ignore_error
:
212 raise CalledProcessError(retcode
, cmd
)
217 """Specifically invoke p4 as the system command. """
218 real_cmd
= p4_build_cmd(cmd
)
219 expand
= isinstance(real_cmd
, basestring
)
220 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
222 raise CalledProcessError(retcode
, real_cmd
)
224 _p4_version_string
= None
225 def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
234 global _p4_version_string
235 if not _p4_version_string
:
236 a
= p4_read_pipe_lines(["-V"])
237 _p4_version_string
= a
[-1].rstrip()
238 return _p4_version_string
240 def p4_integrate(src
, dest
):
241 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
243 def p4_sync(f
, *options
):
244 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
247 # forcibly add file names with wildcards
248 if wildcard_present(f
):
249 p4_system(["add", "-f", f
])
251 p4_system(["add", f
])
254 p4_system(["delete", wildcard_encode(f
)])
256 def p4_edit(f
, *options
):
257 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
260 p4_system(["revert", wildcard_encode(f
)])
262 def p4_reopen(type, f
):
263 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
265 def p4_reopen_in_change(changelist
, files
):
266 cmd
= ["reopen", "-c", str(changelist
)] + files
269 def p4_move(src
, dest
):
270 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
272 def p4_last_change():
273 results
= p4CmdList(["changes", "-m", "1"])
274 return int(results
[0]['change'])
276 def p4_describe(change
):
277 """Make sure it returns a valid result by checking for
278 the presence of field "time". Return a dict of the
281 ds
= p4CmdList(["describe", "-s", str(change
)])
283 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
287 if "p4ExitCode" in d
:
288 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
291 if d
["code"] == "error":
292 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
295 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
300 # Canonicalize the p4 type and return a tuple of the
301 # base type, plus any modifiers. See "p4 help filetypes"
302 # for a list and explanation.
304 def split_p4_type(p4type
):
306 p4_filetypes_historical
= {
307 "ctempobj": "binary+Sw",
313 "tempobj": "binary+FSw",
314 "ubinary": "binary+F",
315 "uresource": "resource+F",
316 "uxbinary": "binary+Fx",
317 "xbinary": "binary+x",
319 "xtempobj": "binary+Swx",
321 "xunicode": "unicode+x",
324 if p4type
in p4_filetypes_historical
:
325 p4type
= p4_filetypes_historical
[p4type
]
327 s
= p4type
.split("+")
335 # return the raw p4 type of a file (text, text+ko, etc)
338 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
339 return results
[0]['headType']
342 # Given a type base and modifier, return a regexp matching
343 # the keywords that can be expanded in the file
345 def p4_keywords_regexp_for_type(base
, type_mods
):
346 if base
in ("text", "unicode", "binary"):
348 if "ko" in type_mods
:
350 elif "k" in type_mods
:
351 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
355 \$ # Starts with a dollar, followed by...
356 (%s) # one of the keywords, followed by...
357 (:[^$\n]+)? # possibly an old expansion, followed by...
365 # Given a file, return a regexp matching the possible
366 # RCS keywords that will be expanded, or None for files
367 # with kw expansion turned off.
369 def p4_keywords_regexp_for_file(file):
370 if not os
.path
.exists(file):
373 (type_base
, type_mods
) = split_p4_type(p4_type(file))
374 return p4_keywords_regexp_for_type(type_base
, type_mods
)
376 def setP4ExecBit(file, mode
):
377 # Reopens an already open file and changes the execute bit to match
378 # the execute bit setting in the passed in mode.
382 if not isModeExec(mode
):
383 p4Type
= getP4OpenedType(file)
384 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
385 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
386 if p4Type
[-1] == "+":
387 p4Type
= p4Type
[0:-1]
389 p4_reopen(p4Type
, file)
391 def getP4OpenedType(file):
392 # Returns the perforce file type for the given file.
394 result
= p4_read_pipe(["opened", wildcard_encode(file)])
395 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
397 return match
.group(1)
399 die("Could not determine file type for %s (result: '%s')" % (file, result
))
401 # Return the set of all p4 labels
402 def getP4Labels(depotPaths
):
404 if isinstance(depotPaths
,basestring
):
405 depotPaths
= [depotPaths
]
407 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
413 # Return the set of all git tags
416 for line
in read_pipe_lines(["git", "tag"]):
421 def diffTreePattern():
422 # This is a simple generator for the diff tree regex pattern. This could be
423 # a class variable if this and parseDiffTreeEntry were a part of a class.
424 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
428 def parseDiffTreeEntry(entry
):
429 """Parses a single diff tree entry into its component elements.
431 See git-diff-tree(1) manpage for details about the format of the diff
432 output. This method returns a dictionary with the following elements:
434 src_mode - The mode of the source file
435 dst_mode - The mode of the destination file
436 src_sha1 - The sha1 for the source file
437 dst_sha1 - The sha1 fr the destination file
438 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
439 status_score - The score for the status (applicable for 'C' and 'R'
440 statuses). This is None if there is no score.
441 src - The path for the source file.
442 dst - The path for the destination file. This is only present for
443 copy or renames. If it is not present, this is None.
445 If the pattern is not matched, None is returned."""
447 match
= diffTreePattern().next().match(entry
)
450 'src_mode': match
.group(1),
451 'dst_mode': match
.group(2),
452 'src_sha1': match
.group(3),
453 'dst_sha1': match
.group(4),
454 'status': match
.group(5),
455 'status_score': match
.group(6),
456 'src': match
.group(7),
457 'dst': match
.group(10)
461 def isModeExec(mode
):
462 # Returns True if the given git mode represents an executable file,
464 return mode
[-3:] == "755"
466 def isModeExecChanged(src_mode
, dst_mode
):
467 return isModeExec(src_mode
) != isModeExec(dst_mode
)
469 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
471 if isinstance(cmd
,basestring
):
478 cmd
= p4_build_cmd(cmd
)
480 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
482 # Use a temporary file to avoid deadlocks without
483 # subprocess.communicate(), which would put another copy
484 # of stdout into memory.
486 if stdin
is not None:
487 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
488 if isinstance(stdin
,basestring
):
489 stdin_file
.write(stdin
)
492 stdin_file
.write(i
+ '\n')
496 p4
= subprocess
.Popen(cmd
,
499 stdout
=subprocess
.PIPE
)
504 entry
= marshal
.load(p4
.stdout
)
514 entry
["p4ExitCode"] = exitCode
520 list = p4CmdList(cmd
)
526 def p4Where(depotPath
):
527 if not depotPath
.endswith("/"):
529 depotPathLong
= depotPath
+ "..."
530 outputList
= p4CmdList(["where", depotPathLong
])
532 for entry
in outputList
:
533 if "depotFile" in entry
:
534 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
535 # The base path always ends with "/...".
536 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
539 elif "data" in entry
:
540 data
= entry
.get("data")
541 space
= data
.find(" ")
542 if data
[:space
] == depotPath
:
547 if output
["code"] == "error":
551 clientPath
= output
.get("path")
552 elif "data" in output
:
553 data
= output
.get("data")
554 lastSpace
= data
.rfind(" ")
555 clientPath
= data
[lastSpace
+ 1:]
557 if clientPath
.endswith("..."):
558 clientPath
= clientPath
[:-3]
561 def currentGitBranch():
562 retcode
= system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error
=True)
567 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
569 def isValidGitDir(path
):
570 if (os
.path
.exists(path
+ "/HEAD")
571 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
575 def parseRevision(ref
):
576 return read_pipe("git rev-parse %s" % ref
).strip()
578 def branchExists(ref
):
579 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
583 def extractLogMessageFromGitCommit(commit
):
586 ## fixme: title is first line of commit, not 1st paragraph.
588 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
597 def extractSettingsGitLog(log
):
599 for line
in log
.split("\n"):
601 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
605 assignments
= m
.group(1).split (':')
606 for a
in assignments
:
608 key
= vals
[0].strip()
609 val
= ('='.join (vals
[1:])).strip()
610 if val
.endswith ('\"') and val
.startswith('"'):
615 paths
= values
.get("depot-paths")
617 paths
= values
.get("depot-path")
619 values
['depot-paths'] = paths
.split(',')
622 def gitBranchExists(branch
):
623 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
624 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
625 return proc
.wait() == 0;
629 def gitConfig(key
, typeSpecifier
=None):
630 if not _gitConfig
.has_key(key
):
631 cmd
= [ "git", "config" ]
633 cmd
+= [ typeSpecifier
]
635 s
= read_pipe(cmd
, ignore_error
=True)
636 _gitConfig
[key
] = s
.strip()
637 return _gitConfig
[key
]
639 def gitConfigBool(key
):
640 """Return a bool, using git config --bool. It is True only if the
641 variable is set to true, and False if set to false or not present
644 if not _gitConfig
.has_key(key
):
645 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
646 return _gitConfig
[key
]
648 def gitConfigInt(key
):
649 if not _gitConfig
.has_key(key
):
650 cmd
= [ "git", "config", "--int", key
]
651 s
= read_pipe(cmd
, ignore_error
=True)
654 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
656 _gitConfig
[key
] = None
657 return _gitConfig
[key
]
659 def gitConfigList(key
):
660 if not _gitConfig
.has_key(key
):
661 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
662 _gitConfig
[key
] = s
.strip().split(os
.linesep
)
663 if _gitConfig
[key
] == ['']:
665 return _gitConfig
[key
]
667 def p4BranchesInGit(branchesAreInRemotes
=True):
668 """Find all the branches whose names start with "p4/", looking
669 in remotes or heads as specified by the argument. Return
670 a dictionary of { branch: revision } for each one found.
671 The branch names are the short names, without any
676 cmdline
= "git rev-parse --symbolic "
677 if branchesAreInRemotes
:
678 cmdline
+= "--remotes"
680 cmdline
+= "--branches"
682 for line
in read_pipe_lines(cmdline
):
686 if not line
.startswith('p4/'):
688 # special symbolic ref to p4/master
689 if line
== "p4/HEAD":
692 # strip off p4/ prefix
693 branch
= line
[len("p4/"):]
695 branches
[branch
] = parseRevision(line
)
699 def branch_exists(branch
):
700 """Make sure that the given ref name really exists."""
702 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
703 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
704 out
, _
= p
.communicate()
707 # expect exactly one line of output: the branch name
708 return out
.rstrip() == branch
710 def findUpstreamBranchPoint(head
= "HEAD"):
711 branches
= p4BranchesInGit()
712 # map from depot-path to branch name
713 branchByDepotPath
= {}
714 for branch
in branches
.keys():
715 tip
= branches
[branch
]
716 log
= extractLogMessageFromGitCommit(tip
)
717 settings
= extractSettingsGitLog(log
)
718 if settings
.has_key("depot-paths"):
719 paths
= ",".join(settings
["depot-paths"])
720 branchByDepotPath
[paths
] = "remotes/p4/" + branch
724 while parent
< 65535:
725 commit
= head
+ "~%s" % parent
726 log
= extractLogMessageFromGitCommit(commit
)
727 settings
= extractSettingsGitLog(log
)
728 if settings
.has_key("depot-paths"):
729 paths
= ",".join(settings
["depot-paths"])
730 if branchByDepotPath
.has_key(paths
):
731 return [branchByDepotPath
[paths
], settings
]
735 return ["", settings
]
737 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
739 print ("Creating/updating branch(es) in %s based on origin branch(es)"
742 originPrefix
= "origin/p4/"
744 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
746 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
749 headName
= line
[len(originPrefix
):]
750 remoteHead
= localRefPrefix
+ headName
753 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
754 if (not original
.has_key('depot-paths')
755 or not original
.has_key('change')):
759 if not gitBranchExists(remoteHead
):
761 print "creating %s" % remoteHead
764 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
765 if settings
.has_key('change') > 0:
766 if settings
['depot-paths'] == original
['depot-paths']:
767 originP4Change
= int(original
['change'])
768 p4Change
= int(settings
['change'])
769 if originP4Change
> p4Change
:
770 print ("%s (%s) is newer than %s (%s). "
771 "Updating p4 branch from origin."
772 % (originHead
, originP4Change
,
773 remoteHead
, p4Change
))
776 print ("Ignoring: %s was imported from %s while "
777 "%s was imported from %s"
778 % (originHead
, ','.join(original
['depot-paths']),
779 remoteHead
, ','.join(settings
['depot-paths'])))
782 system("git update-ref %s %s" % (remoteHead
, originHead
))
784 def originP4BranchesExist():
785 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
788 def p4ParseNumericChangeRange(parts
):
789 changeStart
= int(parts
[0][1:])
790 if parts
[1] == '#head':
791 changeEnd
= p4_last_change()
793 changeEnd
= int(parts
[1])
795 return (changeStart
, changeEnd
)
797 def chooseBlockSize(blockSize
):
801 return defaultBlockSize
803 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
806 # Parse the change range into start and end. Try to find integer
807 # revision ranges as these can be broken up into blocks to avoid
808 # hitting server-side limits (maxrows, maxscanresults). But if
809 # that doesn't work, fall back to using the raw revision specifier
810 # strings, without using block mode.
812 if changeRange
is None or changeRange
== '':
814 changeEnd
= p4_last_change()
815 block_size
= chooseBlockSize(requestedBlockSize
)
817 parts
= changeRange
.split(',')
818 assert len(parts
) == 2
820 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
821 block_size
= chooseBlockSize(requestedBlockSize
)
823 changeStart
= parts
[0][1:]
825 if requestedBlockSize
:
826 die("cannot use --changes-block-size with non-numeric revisions")
831 # Retrieve changes a block at a time, to prevent running
832 # into a MaxResults/MaxScanRows error from the server.
838 end
= min(changeEnd
, changeStart
+ block_size
)
839 revisionRange
= "%d,%d" % (changeStart
, end
)
841 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
844 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
846 # Insert changes in chronological order
847 for line
in reversed(p4_read_pipe_lines(cmd
)):
848 changes
.append(int(line
.split(" ")[1]))
856 changeStart
= end
+ 1
858 changes
= sorted(changes
)
861 def p4PathStartsWith(path
, prefix
):
862 # This method tries to remedy a potential mixed-case issue:
864 # If UserA adds //depot/DirA/file1
865 # and UserB adds //depot/dira/file2
867 # we may or may not have a problem. If you have core.ignorecase=true,
868 # we treat DirA and dira as the same directory
869 if gitConfigBool("core.ignorecase"):
870 return path
.lower().startswith(prefix
.lower())
871 return path
.startswith(prefix
)
874 """Look at the p4 client spec, create a View() object that contains
875 all the mappings, and return it."""
877 specList
= p4CmdList("client -o")
878 if len(specList
) != 1:
879 die('Output from "client -o" is %d lines, expecting 1' %
882 # dictionary of all client parameters
886 client_name
= entry
["Client"]
888 # just the keys that start with "View"
889 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
892 view
= View(client_name
)
894 # append the lines, in order, to the view
895 for view_num
in range(len(view_keys
)):
896 k
= "View%d" % view_num
897 if k
not in view_keys
:
898 die("Expected view key %s missing" % k
)
899 view
.append(entry
[k
])
904 """Grab the client directory."""
906 output
= p4CmdList("client -o")
908 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
911 if "Root" not in entry
:
912 die('Client has no "Root"')
917 # P4 wildcards are not allowed in filenames. P4 complains
918 # if you simply add them, but you can force it with "-f", in
919 # which case it translates them into %xx encoding internally.
921 def wildcard_decode(path
):
922 # Search for and fix just these four characters. Do % last so
923 # that fixing it does not inadvertently create new %-escapes.
924 # Cannot have * in a filename in windows; untested as to
925 # what p4 would do in such a case.
926 if not platform
.system() == "Windows":
927 path
= path
.replace("%2A", "*")
928 path
= path
.replace("%23", "#") \
929 .replace("%40", "@") \
933 def wildcard_encode(path
):
934 # do % first to avoid double-encoding the %s introduced here
935 path
= path
.replace("%", "%25") \
936 .replace("*", "%2A") \
937 .replace("#", "%23") \
941 def wildcard_present(path
):
942 m
= re
.search("[*#@%]", path
)
945 class LargeFileSystem(object):
946 """Base class for large file system support."""
948 def __init__(self
, writeToGitStream
):
949 self
.largeFiles
= set()
950 self
.writeToGitStream
= writeToGitStream
952 def generatePointer(self
, cloneDestination
, contentFile
):
953 """Return the content of a pointer file that is stored in Git instead of
954 the actual content."""
955 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
957 def pushFile(self
, localLargeFile
):
958 """Push the actual content which is not stored in the Git repository to
960 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
962 def hasLargeFileExtension(self
, relPath
):
965 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
969 def generateTempFile(self
, contents
):
970 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
974 return contentFile
.name
976 def exceedsLargeFileThreshold(self
, relPath
, contents
):
977 if gitConfigInt('git-p4.largeFileThreshold'):
978 contentsSize
= sum(len(d
) for d
in contents
)
979 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
981 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
982 contentsSize
= sum(len(d
) for d
in contents
)
983 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
985 contentTempFile
= self
.generateTempFile(contents
)
986 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
987 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
988 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
990 compressedContentsSize
= zf
.infolist()[0].compress_size
991 os
.remove(contentTempFile
)
992 os
.remove(compressedContentFile
.name
)
993 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
997 def addLargeFile(self
, relPath
):
998 self
.largeFiles
.add(relPath
)
1000 def removeLargeFile(self
, relPath
):
1001 self
.largeFiles
.remove(relPath
)
1003 def isLargeFile(self
, relPath
):
1004 return relPath
in self
.largeFiles
1006 def processContent(self
, git_mode
, relPath
, contents
):
1007 """Processes the content of git fast import. This method decides if a
1008 file is stored in the large file system and handles all necessary
1010 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1011 contentTempFile
= self
.generateTempFile(contents
)
1012 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1013 if pointer_git_mode
:
1014 git_mode
= pointer_git_mode
1016 # Move temp file to final location in large file system
1017 largeFileDir
= os
.path
.dirname(localLargeFile
)
1018 if not os
.path
.isdir(largeFileDir
):
1019 os
.makedirs(largeFileDir
)
1020 shutil
.move(contentTempFile
, localLargeFile
)
1021 self
.addLargeFile(relPath
)
1022 if gitConfigBool('git-p4.largeFilePush'):
1023 self
.pushFile(localLargeFile
)
1025 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1026 return (git_mode
, contents
)
1028 class MockLFS(LargeFileSystem
):
1029 """Mock large file system for testing."""
1031 def generatePointer(self
, contentFile
):
1032 """The pointer content is the original content prefixed with "pointer-".
1033 The local filename of the large file storage is derived from the file content.
1035 with
open(contentFile
, 'r') as f
:
1038 pointerContents
= 'pointer-' + content
1039 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1040 return (gitMode
, pointerContents
, localLargeFile
)
1042 def pushFile(self
, localLargeFile
):
1043 """The remote filename of the large file storage is the same as the local
1044 one but in a different directory.
1046 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1047 if not os
.path
.exists(remotePath
):
1048 os
.makedirs(remotePath
)
1049 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1051 class GitLFS(LargeFileSystem
):
1052 """Git LFS as backend for the git-p4 large file system.
1053 See https://git-lfs.github.com/ for details."""
1055 def __init__(self
, *args
):
1056 LargeFileSystem
.__init
__(self
, *args
)
1057 self
.baseGitAttributes
= []
1059 def generatePointer(self
, contentFile
):
1060 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1061 mode and content which is stored in the Git repository instead of
1062 the actual content. Return also the new location of the actual
1065 if os
.path
.getsize(contentFile
) == 0:
1066 return (None, '', None)
1068 pointerProcess
= subprocess
.Popen(
1069 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1070 stdout
=subprocess
.PIPE
1072 pointerFile
= pointerProcess
.stdout
.read()
1073 if pointerProcess
.wait():
1074 os
.remove(contentFile
)
1075 die('git-lfs pointer command failed. Did you install the extension?')
1077 # Git LFS removed the preamble in the output of the 'pointer' command
1078 # starting from version 1.2.0. Check for the preamble here to support
1080 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1081 if pointerFile
.startswith('Git LFS pointer for'):
1082 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1084 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1085 localLargeFile
= os
.path
.join(
1087 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1090 # LFS Spec states that pointer files should not have the executable bit set.
1092 return (gitMode
, pointerFile
, localLargeFile
)
1094 def pushFile(self
, localLargeFile
):
1095 uploadProcess
= subprocess
.Popen(
1096 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1098 if uploadProcess
.wait():
1099 die('git-lfs push command failed. Did you define a remote?')
1101 def generateGitAttributes(self
):
1103 self
.baseGitAttributes
+
1107 '# Git LFS (see https://git-lfs.github.com/)\n',
1110 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1111 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1113 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1114 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1118 def addLargeFile(self
, relPath
):
1119 LargeFileSystem
.addLargeFile(self
, relPath
)
1120 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1122 def removeLargeFile(self
, relPath
):
1123 LargeFileSystem
.removeLargeFile(self
, relPath
)
1124 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1126 def processContent(self
, git_mode
, relPath
, contents
):
1127 if relPath
== '.gitattributes':
1128 self
.baseGitAttributes
= contents
1129 return (git_mode
, self
.generateGitAttributes())
1131 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1135 self
.usage
= "usage: %prog [options]"
1136 self
.needsGit
= True
1137 self
.verbose
= False
1141 self
.userMapFromPerforceServer
= False
1142 self
.myP4UserId
= None
1146 return self
.myP4UserId
1148 results
= p4CmdList("user -o")
1150 if r
.has_key('User'):
1151 self
.myP4UserId
= r
['User']
1153 die("Could not find your p4 user id")
1155 def p4UserIsMe(self
, p4User
):
1156 # return True if the given p4 user is actually me
1157 me
= self
.p4UserId()
1158 if not p4User
or p4User
!= me
:
1163 def getUserCacheFilename(self
):
1164 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1165 return home
+ "/.gitp4-usercache.txt"
1167 def getUserMapFromPerforceServer(self
):
1168 if self
.userMapFromPerforceServer
:
1173 for output
in p4CmdList("users"):
1174 if not output
.has_key("User"):
1176 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1177 self
.emails
[output
["Email"]] = output
["User"]
1179 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1180 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1181 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1182 if mapUser
and len(mapUser
[0]) == 3:
1183 user
= mapUser
[0][0]
1184 fullname
= mapUser
[0][1]
1185 email
= mapUser
[0][2]
1186 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1187 self
.emails
[email
] = user
1190 for (key
, val
) in self
.users
.items():
1191 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1193 open(self
.getUserCacheFilename(), "wb").write(s
)
1194 self
.userMapFromPerforceServer
= True
1196 def loadUserMapFromCache(self
):
1198 self
.userMapFromPerforceServer
= False
1200 cache
= open(self
.getUserCacheFilename(), "rb")
1201 lines
= cache
.readlines()
1204 entry
= line
.strip().split("\t")
1205 self
.users
[entry
[0]] = entry
[1]
1207 self
.getUserMapFromPerforceServer()
1209 class P4Debug(Command
):
1211 Command
.__init
__(self
)
1213 self
.description
= "A tool to debug the output of p4 -G."
1214 self
.needsGit
= False
1216 def run(self
, args
):
1218 for output
in p4CmdList(args
):
1219 print 'Element: %d' % j
1224 class P4RollBack(Command
):
1226 Command
.__init
__(self
)
1228 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1230 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1231 self
.rollbackLocalBranches
= False
1233 def run(self
, args
):
1236 maxChange
= int(args
[0])
1238 if "p4ExitCode" in p4Cmd("changes -m 1"):
1239 die("Problems executing p4");
1241 if self
.rollbackLocalBranches
:
1242 refPrefix
= "refs/heads/"
1243 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1245 refPrefix
= "refs/remotes/"
1246 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1249 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1251 ref
= refPrefix
+ line
1252 log
= extractLogMessageFromGitCommit(ref
)
1253 settings
= extractSettingsGitLog(log
)
1255 depotPaths
= settings
['depot-paths']
1256 change
= settings
['change']
1260 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1261 for p
in depotPaths
]))) == 0:
1262 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1263 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1266 while change
and int(change
) > maxChange
:
1269 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1270 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1271 log
= extractLogMessageFromGitCommit(ref
)
1272 settings
= extractSettingsGitLog(log
)
1275 depotPaths
= settings
['depot-paths']
1276 change
= settings
['change']
1279 print "%s rewound to %s" % (ref
, change
)
1283 class P4Submit(Command
, P4UserMap
):
1285 conflict_behavior_choices
= ("ask", "skip", "quit")
1288 Command
.__init
__(self
)
1289 P4UserMap
.__init
__(self
)
1291 optparse
.make_option("--origin", dest
="origin"),
1292 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1293 # preserve the user, requires relevant p4 permissions
1294 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1295 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1296 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1297 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1298 optparse
.make_option("--conflict", dest
="conflict_behavior",
1299 choices
=self
.conflict_behavior_choices
),
1300 optparse
.make_option("--branch", dest
="branch"),
1301 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1302 help="Shelve instead of submit. Shelved files are reverted, "
1303 "restoring the workspace to the state before the shelve"),
1304 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="store", type="int",
1305 metavar
="CHANGELIST",
1306 help="update an existing shelved changelist, implies --shelve")
1308 self
.description
= "Submit changes from git to the perforce depot."
1309 self
.usage
+= " [name of git branch to submit into perforce depot]"
1311 self
.detectRenames
= False
1312 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1313 self
.dry_run
= False
1315 self
.update_shelve
= None
1316 self
.prepare_p4_only
= False
1317 self
.conflict_behavior
= None
1318 self
.isWindows
= (platform
.system() == "Windows")
1319 self
.exportLabels
= False
1320 self
.p4HasMoveCommand
= p4_has_move_command()
1323 if gitConfig('git-p4.largeFileSystem'):
1324 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1327 if len(p4CmdList("opened ...")) > 0:
1328 die("You have files opened with perforce! Close them before starting the sync.")
1330 def separate_jobs_from_description(self
, message
):
1331 """Extract and return a possible Jobs field in the commit
1332 message. It goes into a separate section in the p4 change
1335 A jobs line starts with "Jobs:" and looks like a new field
1336 in a form. Values are white-space separated on the same
1337 line or on following lines that start with a tab.
1339 This does not parse and extract the full git commit message
1340 like a p4 form. It just sees the Jobs: line as a marker
1341 to pass everything from then on directly into the p4 form,
1342 but outside the description section.
1344 Return a tuple (stripped log message, jobs string)."""
1346 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1348 return (message
, None)
1350 jobtext
= message
[m
.start():]
1351 stripped_message
= message
[:m
.start()].rstrip()
1352 return (stripped_message
, jobtext
)
1354 def prepareLogMessage(self
, template
, message
, jobs
):
1355 """Edits the template returned from "p4 change -o" to insert
1356 the message in the Description field, and the jobs text in
1360 inDescriptionSection
= False
1362 for line
in template
.split("\n"):
1363 if line
.startswith("#"):
1364 result
+= line
+ "\n"
1367 if inDescriptionSection
:
1368 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1369 inDescriptionSection
= False
1370 # insert Jobs section
1372 result
+= jobs
+ "\n"
1376 if line
.startswith("Description:"):
1377 inDescriptionSection
= True
1379 for messageLine
in message
.split("\n"):
1380 line
+= "\t" + messageLine
+ "\n"
1382 result
+= line
+ "\n"
1386 def patchRCSKeywords(self
, file, pattern
):
1387 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1388 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1390 outFile
= os
.fdopen(handle
, "w+")
1391 inFile
= open(file, "r")
1392 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1393 for line
in inFile
.readlines():
1394 line
= regexp
.sub(r
'$\1$', line
)
1398 # Forcibly overwrite the original file
1400 shutil
.move(outFileName
, file)
1402 # cleanup our temporary file
1403 os
.unlink(outFileName
)
1404 print "Failed to strip RCS keywords in %s" % file
1407 print "Patched up RCS keywords in %s" % file
1409 def p4UserForCommit(self
,id):
1410 # Return the tuple (perforce user,git email) for a given git commit id
1411 self
.getUserMapFromPerforceServer()
1412 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1413 "--format=%ae", id])
1414 gitEmail
= gitEmail
.strip()
1415 if not self
.emails
.has_key(gitEmail
):
1416 return (None,gitEmail
)
1418 return (self
.emails
[gitEmail
],gitEmail
)
1420 def checkValidP4Users(self
,commits
):
1421 # check if any git authors cannot be mapped to p4 users
1423 (user
,email
) = self
.p4UserForCommit(id)
1425 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1426 if gitConfigBool("git-p4.allowMissingP4Users"):
1429 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1431 def lastP4Changelist(self
):
1432 # Get back the last changelist number submitted in this client spec. This
1433 # then gets used to patch up the username in the change. If the same
1434 # client spec is being used by multiple processes then this might go
1436 results
= p4CmdList("client -o") # find the current client
1439 if r
.has_key('Client'):
1440 client
= r
['Client']
1443 die("could not get client spec")
1444 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1446 if r
.has_key('change'):
1448 die("Could not get changelist number for last submit - cannot patch up user details")
1450 def modifyChangelistUser(self
, changelist
, newUser
):
1451 # fixup the user field of a changelist after it has been submitted.
1452 changes
= p4CmdList("change -o %s" % changelist
)
1453 if len(changes
) != 1:
1454 die("Bad output from p4 change modifying %s to user %s" %
1455 (changelist
, newUser
))
1458 if c
['User'] == newUser
: return # nothing to do
1460 input = marshal
.dumps(c
)
1462 result
= p4CmdList("change -f -i", stdin
=input)
1464 if r
.has_key('code'):
1465 if r
['code'] == 'error':
1466 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1467 if r
.has_key('data'):
1468 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1470 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1472 def canChangeChangelists(self
):
1473 # check to see if we have p4 admin or super-user permissions, either of
1474 # which are required to modify changelists.
1475 results
= p4CmdList(["protects", self
.depotPath
])
1477 if r
.has_key('perm'):
1478 if r
['perm'] == 'admin':
1480 if r
['perm'] == 'super':
1484 def prepareSubmitTemplate(self
, changelist
=None):
1485 """Run "p4 change -o" to grab a change specification template.
1486 This does not use "p4 -G", as it is nice to keep the submission
1487 template in original order, since a human might edit it.
1489 Remove lines in the Files section that show changes to files
1490 outside the depot path we're committing into."""
1492 [upstream
, settings
] = findUpstreamBranchPoint()
1495 inFilesSection
= False
1496 args
= ['change', '-o']
1498 args
.append(str(changelist
))
1500 for line
in p4_read_pipe_lines(args
):
1501 if line
.endswith("\r\n"):
1502 line
= line
[:-2] + "\n"
1504 if line
.startswith("\t"):
1505 # path starts and ends with a tab
1507 lastTab
= path
.rfind("\t")
1509 path
= path
[:lastTab
]
1510 if settings
.has_key('depot-paths'):
1511 if not [p
for p
in settings
['depot-paths']
1512 if p4PathStartsWith(path
, p
)]:
1515 if not p4PathStartsWith(path
, self
.depotPath
):
1518 inFilesSection
= False
1520 if line
.startswith("Files:"):
1521 inFilesSection
= True
1527 def edit_template(self
, template_file
):
1528 """Invoke the editor to let the user change the submission
1529 message. Return true if okay to continue with the submit."""
1531 # if configured to skip the editing part, just submit
1532 if gitConfigBool("git-p4.skipSubmitEdit"):
1535 # look at the modification time, to check later if the user saved
1537 mtime
= os
.stat(template_file
).st_mtime
1540 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1541 editor
= os
.environ
.get("P4EDITOR")
1543 editor
= read_pipe("git var GIT_EDITOR").strip()
1544 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1546 # If the file was not saved, prompt to see if this patch should
1547 # be skipped. But skip this verification step if configured so.
1548 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1551 # modification time updated means user saved the file
1552 if os
.stat(template_file
).st_mtime
> mtime
:
1556 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1562 def get_diff_description(self
, editedFiles
, filesToAdd
):
1564 if os
.environ
.has_key("P4DIFF"):
1565 del(os
.environ
["P4DIFF"])
1567 for editedFile
in editedFiles
:
1568 diff
+= p4_read_pipe(['diff', '-du',
1569 wildcard_encode(editedFile
)])
1573 for newFile
in filesToAdd
:
1574 newdiff
+= "==== new file ====\n"
1575 newdiff
+= "--- /dev/null\n"
1576 newdiff
+= "+++ %s\n" % newFile
1577 f
= open(newFile
, "r")
1578 for line
in f
.readlines():
1579 newdiff
+= "+" + line
1582 return (diff
+ newdiff
).replace('\r\n', '\n')
1584 def applyCommit(self
, id):
1585 """Apply one commit, return True if it succeeded."""
1587 print "Applying", read_pipe(["git", "show", "-s",
1588 "--format=format:%h %s", id])
1590 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1592 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1594 filesToChangeType
= set()
1595 filesToDelete
= set()
1597 pureRenameCopy
= set()
1598 filesToChangeExecBit
= {}
1602 diff
= parseDiffTreeEntry(line
)
1603 modifier
= diff
['status']
1605 all_files
.append(path
)
1609 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1610 filesToChangeExecBit
[path
] = diff
['dst_mode']
1611 editedFiles
.add(path
)
1612 elif modifier
== "A":
1613 filesToAdd
.add(path
)
1614 filesToChangeExecBit
[path
] = diff
['dst_mode']
1615 if path
in filesToDelete
:
1616 filesToDelete
.remove(path
)
1617 elif modifier
== "D":
1618 filesToDelete
.add(path
)
1619 if path
in filesToAdd
:
1620 filesToAdd
.remove(path
)
1621 elif modifier
== "C":
1622 src
, dest
= diff
['src'], diff
['dst']
1623 p4_integrate(src
, dest
)
1624 pureRenameCopy
.add(dest
)
1625 if diff
['src_sha1'] != diff
['dst_sha1']:
1627 pureRenameCopy
.discard(dest
)
1628 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1630 pureRenameCopy
.discard(dest
)
1631 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1633 # turn off read-only attribute
1634 os
.chmod(dest
, stat
.S_IWRITE
)
1636 editedFiles
.add(dest
)
1637 elif modifier
== "R":
1638 src
, dest
= diff
['src'], diff
['dst']
1639 if self
.p4HasMoveCommand
:
1640 p4_edit(src
) # src must be open before move
1641 p4_move(src
, dest
) # opens for (move/delete, move/add)
1643 p4_integrate(src
, dest
)
1644 if diff
['src_sha1'] != diff
['dst_sha1']:
1647 pureRenameCopy
.add(dest
)
1648 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1649 if not self
.p4HasMoveCommand
:
1650 p4_edit(dest
) # with move: already open, writable
1651 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1652 if not self
.p4HasMoveCommand
:
1654 os
.chmod(dest
, stat
.S_IWRITE
)
1656 filesToDelete
.add(src
)
1657 editedFiles
.add(dest
)
1658 elif modifier
== "T":
1659 filesToChangeType
.add(path
)
1661 die("unknown modifier %s for %s" % (modifier
, path
))
1663 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1664 patchcmd
= diffcmd
+ " | git apply "
1665 tryPatchCmd
= patchcmd
+ "--check -"
1666 applyPatchCmd
= patchcmd
+ "--check --apply -"
1667 patch_succeeded
= True
1669 if os
.system(tryPatchCmd
) != 0:
1670 fixed_rcs_keywords
= False
1671 patch_succeeded
= False
1672 print "Unfortunately applying the change failed!"
1674 # Patch failed, maybe it's just RCS keyword woes. Look through
1675 # the patch to see if that's possible.
1676 if gitConfigBool("git-p4.attemptRCSCleanup"):
1680 for file in editedFiles | filesToDelete
:
1681 # did this file's delta contain RCS keywords?
1682 pattern
= p4_keywords_regexp_for_file(file)
1685 # this file is a possibility...look for RCS keywords.
1686 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1687 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1688 if regexp
.search(line
):
1690 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1691 kwfiles
[file] = pattern
1694 for file in kwfiles
:
1696 print "zapping %s with %s" % (line
,pattern
)
1697 # File is being deleted, so not open in p4. Must
1698 # disable the read-only bit on windows.
1699 if self
.isWindows
and file not in editedFiles
:
1700 os
.chmod(file, stat
.S_IWRITE
)
1701 self
.patchRCSKeywords(file, kwfiles
[file])
1702 fixed_rcs_keywords
= True
1704 if fixed_rcs_keywords
:
1705 print "Retrying the patch with RCS keywords cleaned up"
1706 if os
.system(tryPatchCmd
) == 0:
1707 patch_succeeded
= True
1709 if not patch_succeeded
:
1710 for f
in editedFiles
:
1715 # Apply the patch for real, and do add/delete/+x handling.
1717 system(applyPatchCmd
)
1719 for f
in filesToChangeType
:
1720 p4_edit(f
, "-t", "auto")
1721 for f
in filesToAdd
:
1723 for f
in filesToDelete
:
1727 # Set/clear executable bits
1728 for f
in filesToChangeExecBit
.keys():
1729 mode
= filesToChangeExecBit
[f
]
1730 setP4ExecBit(f
, mode
)
1732 if self
.update_shelve
:
1733 print("all_files = %s" % str(all_files
))
1734 p4_reopen_in_change(self
.update_shelve
, all_files
)
1737 # Build p4 change description, starting with the contents
1738 # of the git commit message.
1740 logMessage
= extractLogMessageFromGitCommit(id)
1741 logMessage
= logMessage
.strip()
1742 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1744 template
= self
.prepareSubmitTemplate(self
.update_shelve
)
1745 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1747 if self
.preserveUser
:
1748 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1750 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1751 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1752 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1753 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1755 separatorLine
= "######## everything below this line is just the diff #######\n"
1756 if not self
.prepare_p4_only
:
1757 submitTemplate
+= separatorLine
1758 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
)
1760 (handle
, fileName
) = tempfile
.mkstemp()
1761 tmpFile
= os
.fdopen(handle
, "w+b")
1763 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1764 tmpFile
.write(submitTemplate
)
1767 if self
.prepare_p4_only
:
1769 # Leave the p4 tree prepared, and the submit template around
1770 # and let the user decide what to do next
1773 print "P4 workspace prepared for submission."
1774 print "To submit or revert, go to client workspace"
1775 print " " + self
.clientPath
1777 print "To submit, use \"p4 submit\" to write a new description,"
1778 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1779 " \"git p4\"." % fileName
1780 print "You can delete the file \"%s\" when finished." % fileName
1782 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1783 print "To preserve change ownership by user %s, you must\n" \
1784 "do \"p4 change -f <change>\" after submitting and\n" \
1785 "edit the User field."
1787 print "After submitting, renamed files must be re-synced."
1788 print "Invoke \"p4 sync -f\" on each of these files:"
1789 for f
in pureRenameCopy
:
1793 print "To revert the changes, use \"p4 revert ...\", and delete"
1794 print "the submit template file \"%s\"" % fileName
1796 print "Since the commit adds new files, they must be deleted:"
1797 for f
in filesToAdd
:
1803 # Let the user edit the change description, then submit it.
1808 if self
.edit_template(fileName
):
1809 # read the edited message and submit
1810 tmpFile
= open(fileName
, "rb")
1811 message
= tmpFile
.read()
1814 message
= message
.replace("\r\n", "\n")
1815 submitTemplate
= message
[:message
.index(separatorLine
)]
1817 if self
.update_shelve
:
1818 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
1820 p4_write_pipe(['shelve', '-i'], submitTemplate
)
1822 p4_write_pipe(['submit', '-i'], submitTemplate
)
1823 # The rename/copy happened by applying a patch that created a
1824 # new file. This leaves it writable, which confuses p4.
1825 for f
in pureRenameCopy
:
1828 if self
.preserveUser
:
1830 # Get last changelist number. Cannot easily get it from
1831 # the submit command output as the output is
1833 changelist
= self
.lastP4Changelist()
1834 self
.modifyChangelistUser(changelist
, p4User
)
1840 if not submitted
or self
.shelve
:
1842 print ("Reverting shelved files.")
1844 print ("Submission cancelled, undoing p4 changes.")
1845 for f
in editedFiles | filesToDelete
:
1847 for f
in filesToAdd
:
1854 # Export git tags as p4 labels. Create a p4 label and then tag
1856 def exportGitTags(self
, gitTags
):
1857 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1858 if len(validLabelRegexp
) == 0:
1859 validLabelRegexp
= defaultLabelRegexp
1860 m
= re
.compile(validLabelRegexp
)
1862 for name
in gitTags
:
1864 if not m
.match(name
):
1866 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1869 # Get the p4 commit this corresponds to
1870 logMessage
= extractLogMessageFromGitCommit(name
)
1871 values
= extractSettingsGitLog(logMessage
)
1873 if not values
.has_key('change'):
1874 # a tag pointing to something not sent to p4; ignore
1876 print "git tag %s does not give a p4 commit" % name
1879 changelist
= values
['change']
1881 # Get the tag details.
1885 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1888 if re
.match(r
'tag\s+', l
):
1890 elif re
.match(r
'\s*$', l
):
1897 body
= ["lightweight tag imported by git p4\n"]
1899 # Create the label - use the same view as the client spec we are using
1900 clientSpec
= getClientSpec()
1902 labelTemplate
= "Label: %s\n" % name
1903 labelTemplate
+= "Description:\n"
1905 labelTemplate
+= "\t" + b
+ "\n"
1906 labelTemplate
+= "View:\n"
1907 for depot_side
in clientSpec
.mappings
:
1908 labelTemplate
+= "\t%s\n" % depot_side
1911 print "Would create p4 label %s for tag" % name
1912 elif self
.prepare_p4_only
:
1913 print "Not creating p4 label %s for tag due to option" \
1914 " --prepare-p4-only" % name
1916 p4_write_pipe(["label", "-i"], labelTemplate
)
1919 p4_system(["tag", "-l", name
] +
1920 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
1923 print "created p4 label for tag %s" % name
1925 def run(self
, args
):
1927 self
.master
= currentGitBranch()
1928 elif len(args
) == 1:
1929 self
.master
= args
[0]
1930 if not branchExists(self
.master
):
1931 die("Branch %s does not exist" % self
.master
)
1936 allowSubmit
= gitConfig("git-p4.allowSubmit")
1937 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1938 die("%s is not in git-p4.allowSubmit" % self
.master
)
1940 [upstream
, settings
] = findUpstreamBranchPoint()
1941 self
.depotPath
= settings
['depot-paths'][0]
1942 if len(self
.origin
) == 0:
1943 self
.origin
= upstream
1945 if self
.update_shelve
:
1948 if self
.preserveUser
:
1949 if not self
.canChangeChangelists():
1950 die("Cannot preserve user names without p4 super-user or admin permissions")
1952 # if not set from the command line, try the config file
1953 if self
.conflict_behavior
is None:
1954 val
= gitConfig("git-p4.conflict")
1956 if val
not in self
.conflict_behavior_choices
:
1957 die("Invalid value '%s' for config git-p4.conflict" % val
)
1960 self
.conflict_behavior
= val
1963 print "Origin branch is " + self
.origin
1965 if len(self
.depotPath
) == 0:
1966 print "Internal error: cannot locate perforce depot path from existing branches"
1969 self
.useClientSpec
= False
1970 if gitConfigBool("git-p4.useclientspec"):
1971 self
.useClientSpec
= True
1972 if self
.useClientSpec
:
1973 self
.clientSpecDirs
= getClientSpec()
1975 # Check for the existence of P4 branches
1976 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
1978 if self
.useClientSpec
and not branchesDetected
:
1979 # all files are relative to the client spec
1980 self
.clientPath
= getClientRoot()
1982 self
.clientPath
= p4Where(self
.depotPath
)
1984 if self
.clientPath
== "":
1985 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1987 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1988 self
.oldWorkingDirectory
= os
.getcwd()
1990 # ensure the clientPath exists
1991 new_client_dir
= False
1992 if not os
.path
.exists(self
.clientPath
):
1993 new_client_dir
= True
1994 os
.makedirs(self
.clientPath
)
1996 chdir(self
.clientPath
, is_client_path
=True)
1998 print "Would synchronize p4 checkout in %s" % self
.clientPath
2000 print "Synchronizing p4 checkout..."
2002 # old one was destroyed, and maybe nobody told p4
2003 p4_sync("...", "-f")
2010 commitish
= self
.master
2014 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, commitish
)]):
2015 commits
.append(line
.strip())
2018 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2019 self
.checkAuthorship
= False
2021 self
.checkAuthorship
= True
2023 if self
.preserveUser
:
2024 self
.checkValidP4Users(commits
)
2027 # Build up a set of options to be passed to diff when
2028 # submitting each commit to p4.
2030 if self
.detectRenames
:
2031 # command-line -M arg
2032 self
.diffOpts
= "-M"
2034 # If not explicitly set check the config variable
2035 detectRenames
= gitConfig("git-p4.detectRenames")
2037 if detectRenames
.lower() == "false" or detectRenames
== "":
2039 elif detectRenames
.lower() == "true":
2040 self
.diffOpts
= "-M"
2042 self
.diffOpts
= "-M%s" % detectRenames
2044 # no command-line arg for -C or --find-copies-harder, just
2046 detectCopies
= gitConfig("git-p4.detectCopies")
2047 if detectCopies
.lower() == "false" or detectCopies
== "":
2049 elif detectCopies
.lower() == "true":
2050 self
.diffOpts
+= " -C"
2052 self
.diffOpts
+= " -C%s" % detectCopies
2054 if gitConfigBool("git-p4.detectCopiesHarder"):
2055 self
.diffOpts
+= " --find-copies-harder"
2058 # Apply the commits, one at a time. On failure, ask if should
2059 # continue to try the rest of the patches, or quit.
2064 last
= len(commits
) - 1
2065 for i
, commit
in enumerate(commits
):
2067 print " ", read_pipe(["git", "show", "-s",
2068 "--format=format:%h %s", commit
])
2071 ok
= self
.applyCommit(commit
)
2073 applied
.append(commit
)
2075 if self
.prepare_p4_only
and i
< last
:
2076 print "Processing only the first commit due to option" \
2077 " --prepare-p4-only"
2082 # prompt for what to do, or use the option/variable
2083 if self
.conflict_behavior
== "ask":
2084 print "What do you want to do?"
2085 response
= raw_input("[s]kip this commit but apply"
2086 " the rest, or [q]uit? ")
2089 elif self
.conflict_behavior
== "skip":
2091 elif self
.conflict_behavior
== "quit":
2094 die("Unknown conflict_behavior '%s'" %
2095 self
.conflict_behavior
)
2097 if response
[0] == "s":
2098 print "Skipping this commit, but applying the rest"
2100 if response
[0] == "q":
2107 chdir(self
.oldWorkingDirectory
)
2108 shelved_applied
= "shelved" if self
.shelve
else "applied"
2111 elif self
.prepare_p4_only
:
2113 elif len(commits
) == len(applied
):
2114 print ("All commits {0}!".format(shelved_applied
))
2118 sync
.branch
= self
.branch
2125 if len(applied
) == 0:
2126 print ("No commits {0}.".format(shelved_applied
))
2128 print ("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2134 print star
, read_pipe(["git", "show", "-s",
2135 "--format=format:%h %s", c
])
2136 print "You will have to do 'git p4 sync' and rebase."
2138 if gitConfigBool("git-p4.exportLabels"):
2139 self
.exportLabels
= True
2141 if self
.exportLabels
:
2142 p4Labels
= getP4Labels(self
.depotPath
)
2143 gitTags
= getGitTags()
2145 missingGitTags
= gitTags
- p4Labels
2146 self
.exportGitTags(missingGitTags
)
2148 # exit with error unless everything applied perfectly
2149 if len(commits
) != len(applied
):
2155 """Represent a p4 view ("p4 help views"), and map files in a
2156 repo according to the view."""
2158 def __init__(self
, client_name
):
2160 self
.client_prefix
= "//%s/" % client_name
2161 # cache results of "p4 where" to lookup client file locations
2162 self
.client_spec_path_cache
= {}
2164 def append(self
, view_line
):
2165 """Parse a view line, splitting it into depot and client
2166 sides. Append to self.mappings, preserving order. This
2167 is only needed for tag creation."""
2169 # Split the view line into exactly two words. P4 enforces
2170 # structure on these lines that simplifies this quite a bit.
2172 # Either or both words may be double-quoted.
2173 # Single quotes do not matter.
2174 # Double-quote marks cannot occur inside the words.
2175 # A + or - prefix is also inside the quotes.
2176 # There are no quotes unless they contain a space.
2177 # The line is already white-space stripped.
2178 # The two words are separated by a single space.
2180 if view_line
[0] == '"':
2181 # First word is double quoted. Find its end.
2182 close_quote_index
= view_line
.find('"', 1)
2183 if close_quote_index
<= 0:
2184 die("No first-word closing quote found: %s" % view_line
)
2185 depot_side
= view_line
[1:close_quote_index
]
2186 # skip closing quote and space
2187 rhs_index
= close_quote_index
+ 1 + 1
2189 space_index
= view_line
.find(" ")
2190 if space_index
<= 0:
2191 die("No word-splitting space found: %s" % view_line
)
2192 depot_side
= view_line
[0:space_index
]
2193 rhs_index
= space_index
+ 1
2195 # prefix + means overlay on previous mapping
2196 if depot_side
.startswith("+"):
2197 depot_side
= depot_side
[1:]
2199 # prefix - means exclude this path, leave out of mappings
2201 if depot_side
.startswith("-"):
2203 depot_side
= depot_side
[1:]
2206 self
.mappings
.append(depot_side
)
2208 def convert_client_path(self
, clientFile
):
2209 # chop off //client/ part to make it relative
2210 if not clientFile
.startswith(self
.client_prefix
):
2211 die("No prefix '%s' on clientFile '%s'" %
2212 (self
.client_prefix
, clientFile
))
2213 return clientFile
[len(self
.client_prefix
):]
2215 def update_client_spec_path_cache(self
, files
):
2216 """ Caching file paths by "p4 where" batch query """
2218 # List depot file paths exclude that already cached
2219 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2221 if len(fileArgs
) == 0:
2222 return # All files in cache
2224 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2225 for res
in where_result
:
2226 if "code" in res
and res
["code"] == "error":
2227 # assume error is "... file(s) not in client view"
2229 if "clientFile" not in res
:
2230 die("No clientFile in 'p4 where' output")
2232 # it will list all of them, but only one not unmap-ped
2234 if gitConfigBool("core.ignorecase"):
2235 res
['depotFile'] = res
['depotFile'].lower()
2236 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2238 # not found files or unmap files set to ""
2239 for depotFile
in fileArgs
:
2240 if gitConfigBool("core.ignorecase"):
2241 depotFile
= depotFile
.lower()
2242 if depotFile
not in self
.client_spec_path_cache
:
2243 self
.client_spec_path_cache
[depotFile
] = ""
2245 def map_in_client(self
, depot_path
):
2246 """Return the relative location in the client where this
2247 depot file should live. Returns "" if the file should
2248 not be mapped in the client."""
2250 if gitConfigBool("core.ignorecase"):
2251 depot_path
= depot_path
.lower()
2253 if depot_path
in self
.client_spec_path_cache
:
2254 return self
.client_spec_path_cache
[depot_path
]
2256 die( "Error: %s is not found in client spec path" % depot_path
)
2259 class P4Sync(Command
, P4UserMap
):
2260 delete_actions
= ( "delete", "move/delete", "purge" )
2263 Command
.__init
__(self
)
2264 P4UserMap
.__init
__(self
)
2266 optparse
.make_option("--branch", dest
="branch"),
2267 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2268 optparse
.make_option("--changesfile", dest
="changesFile"),
2269 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2270 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2271 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2272 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2273 help="Import into refs/heads/ , not refs/remotes"),
2274 optparse
.make_option("--max-changes", dest
="maxChanges",
2275 help="Maximum number of changes to import"),
2276 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2277 help="Internal block size to use when iteratively calling p4 changes"),
2278 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2279 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2280 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2281 help="Only sync files that are included in the Perforce Client Spec"),
2282 optparse
.make_option("-/", dest
="cloneExclude",
2283 action
="append", type="string",
2284 help="exclude depot path"),
2286 self
.description
= """Imports from Perforce into a git repository.\n
2288 //depot/my/project/ -- to import the current head
2289 //depot/my/project/@all -- to import everything
2290 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2292 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2294 self
.usage
+= " //depot/path[@revRange]"
2296 self
.createdBranches
= set()
2297 self
.committedChanges
= set()
2299 self
.detectBranches
= False
2300 self
.detectLabels
= False
2301 self
.importLabels
= False
2302 self
.changesFile
= ""
2303 self
.syncWithOrigin
= True
2304 self
.importIntoRemotes
= True
2305 self
.maxChanges
= ""
2306 self
.changes_block_size
= None
2307 self
.keepRepoPath
= False
2308 self
.depotPaths
= None
2309 self
.p4BranchesInGit
= []
2310 self
.cloneExclude
= []
2311 self
.useClientSpec
= False
2312 self
.useClientSpec_from_options
= False
2313 self
.clientSpecDirs
= None
2314 self
.tempBranches
= []
2315 self
.tempBranchLocation
= "refs/git-p4-tmp"
2316 self
.largeFileSystem
= None
2318 if gitConfig('git-p4.largeFileSystem'):
2319 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2320 self
.largeFileSystem
= largeFileSystemConstructor(
2321 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2324 if gitConfig("git-p4.syncFromOrigin") == "false":
2325 self
.syncWithOrigin
= False
2327 # This is required for the "append" cloneExclude action
2328 def ensure_value(self
, attr
, value
):
2329 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2330 setattr(self
, attr
, value
)
2331 return getattr(self
, attr
)
2333 # Force a checkpoint in fast-import and wait for it to finish
2334 def checkpoint(self
):
2335 self
.gitStream
.write("checkpoint\n\n")
2336 self
.gitStream
.write("progress checkpoint\n\n")
2337 out
= self
.gitOutput
.readline()
2339 print "checkpoint finished: " + out
2341 def extractFilesFromCommit(self
, commit
):
2342 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2343 for path
in self
.cloneExclude
]
2346 while commit
.has_key("depotFile%s" % fnum
):
2347 path
= commit
["depotFile%s" % fnum
]
2349 if [p
for p
in self
.cloneExclude
2350 if p4PathStartsWith(path
, p
)]:
2353 found
= [p
for p
in self
.depotPaths
2354 if p4PathStartsWith(path
, p
)]
2361 file["rev"] = commit
["rev%s" % fnum
]
2362 file["action"] = commit
["action%s" % fnum
]
2363 file["type"] = commit
["type%s" % fnum
]
2368 def extractJobsFromCommit(self
, commit
):
2371 while commit
.has_key("job%s" % jnum
):
2372 job
= commit
["job%s" % jnum
]
2377 def stripRepoPath(self
, path
, prefixes
):
2378 """When streaming files, this is called to map a p4 depot path
2379 to where it should go in git. The prefixes are either
2380 self.depotPaths, or self.branchPrefixes in the case of
2381 branch detection."""
2383 if self
.useClientSpec
:
2384 # branch detection moves files up a level (the branch name)
2385 # from what client spec interpretation gives
2386 path
= self
.clientSpecDirs
.map_in_client(path
)
2387 if self
.detectBranches
:
2388 for b
in self
.knownBranches
:
2389 if path
.startswith(b
+ "/"):
2390 path
= path
[len(b
)+1:]
2392 elif self
.keepRepoPath
:
2393 # Preserve everything in relative path name except leading
2394 # //depot/; just look at first prefix as they all should
2395 # be in the same depot.
2396 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2397 if p4PathStartsWith(path
, depot
):
2398 path
= path
[len(depot
):]
2402 if p4PathStartsWith(path
, p
):
2403 path
= path
[len(p
):]
2406 path
= wildcard_decode(path
)
2409 def splitFilesIntoBranches(self
, commit
):
2410 """Look at each depotFile in the commit to figure out to what
2411 branch it belongs."""
2413 if self
.clientSpecDirs
:
2414 files
= self
.extractFilesFromCommit(commit
)
2415 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2419 while commit
.has_key("depotFile%s" % fnum
):
2420 path
= commit
["depotFile%s" % fnum
]
2421 found
= [p
for p
in self
.depotPaths
2422 if p4PathStartsWith(path
, p
)]
2429 file["rev"] = commit
["rev%s" % fnum
]
2430 file["action"] = commit
["action%s" % fnum
]
2431 file["type"] = commit
["type%s" % fnum
]
2434 # start with the full relative path where this file would
2436 if self
.useClientSpec
:
2437 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2439 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2441 for branch
in self
.knownBranches
.keys():
2442 # add a trailing slash so that a commit into qt/4.2foo
2443 # doesn't end up in qt/4.2, e.g.
2444 if relPath
.startswith(branch
+ "/"):
2445 if branch
not in branches
:
2446 branches
[branch
] = []
2447 branches
[branch
].append(file)
2452 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2453 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2454 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2456 self
.gitStream
.write(d
)
2457 self
.gitStream
.write('\n')
2459 # output one file from the P4 stream
2460 # - helper for streamP4Files
2462 def streamOneP4File(self
, file, contents
):
2463 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2465 size
= int(self
.stream_file
['fileSize'])
2466 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2469 (type_base
, type_mods
) = split_p4_type(file["type"])
2472 if "x" in type_mods
:
2474 if type_base
== "symlink":
2476 # p4 print on a symlink sometimes contains "target\n";
2477 # if it does, remove the newline
2478 data
= ''.join(contents
)
2480 # Some version of p4 allowed creating a symlink that pointed
2481 # to nothing. This causes p4 errors when checking out such
2482 # a change, and errors here too. Work around it by ignoring
2483 # the bad symlink; hopefully a future change fixes it.
2484 print "\nIgnoring empty symlink in %s" % file['depotFile']
2486 elif data
[-1] == '\n':
2487 contents
= [data
[:-1]]
2491 if type_base
== "utf16":
2492 # p4 delivers different text in the python output to -G
2493 # than it does when using "print -o", or normal p4 client
2494 # operations. utf16 is converted to ascii or utf8, perhaps.
2495 # But ascii text saved as -t utf16 is completely mangled.
2496 # Invoke print -o to get the real contents.
2498 # On windows, the newlines will always be mangled by print, so put
2499 # them back too. This is not needed to the cygwin windows version,
2500 # just the native "NT" type.
2503 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2504 except Exception as e
:
2505 if 'Translation of file content failed' in str(e
):
2506 type_base
= 'binary'
2510 if p4_version_string().find('/NT') >= 0:
2511 text
= text
.replace('\r\n', '\n')
2514 if type_base
== "apple":
2515 # Apple filetype files will be streamed as a concatenation of
2516 # its appledouble header and the contents. This is useless
2517 # on both macs and non-macs. If using "print -q -o xx", it
2518 # will create "xx" with the data, and "%xx" with the header.
2519 # This is also not very useful.
2521 # Ideally, someday, this script can learn how to generate
2522 # appledouble files directly and import those to git, but
2523 # non-mac machines can never find a use for apple filetype.
2524 print "\nIgnoring apple filetype file %s" % file['depotFile']
2527 # Note that we do not try to de-mangle keywords on utf16 files,
2528 # even though in theory somebody may want that.
2529 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2531 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2532 text
= ''.join(contents
)
2533 text
= regexp
.sub(r
'$\1$', text
)
2537 relPath
.decode('ascii')
2540 if gitConfig('git-p4.pathEncoding'):
2541 encoding
= gitConfig('git-p4.pathEncoding')
2542 relPath
= relPath
.decode(encoding
, 'replace').encode('utf8', 'replace')
2544 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, relPath
)
2546 if self
.largeFileSystem
:
2547 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2549 self
.writeToGitStream(git_mode
, relPath
, contents
)
2551 def streamOneP4Deletion(self
, file):
2552 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2554 sys
.stdout
.write("delete %s\n" % relPath
)
2556 self
.gitStream
.write("D %s\n" % relPath
)
2558 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2559 self
.largeFileSystem
.removeLargeFile(relPath
)
2561 # handle another chunk of streaming data
2562 def streamP4FilesCb(self
, marshalled
):
2564 # catch p4 errors and complain
2566 if "code" in marshalled
:
2567 if marshalled
["code"] == "error":
2568 if "data" in marshalled
:
2569 err
= marshalled
["data"].rstrip()
2571 if not err
and 'fileSize' in self
.stream_file
:
2572 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2573 if required_bytes
> 0:
2574 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2575 os
.getcwd(), required_bytes
/1024/1024
2580 if self
.stream_have_file_info
:
2581 if "depotFile" in self
.stream_file
:
2582 f
= self
.stream_file
["depotFile"]
2583 # force a failure in fast-import, else an empty
2584 # commit will be made
2585 self
.gitStream
.write("\n")
2586 self
.gitStream
.write("die-now\n")
2587 self
.gitStream
.close()
2588 # ignore errors, but make sure it exits first
2589 self
.importProcess
.wait()
2591 die("Error from p4 print for %s: %s" % (f
, err
))
2593 die("Error from p4 print: %s" % err
)
2595 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2596 # start of a new file - output the old one first
2597 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2598 self
.stream_file
= {}
2599 self
.stream_contents
= []
2600 self
.stream_have_file_info
= False
2602 # pick up the new file information... for the
2603 # 'data' field we need to append to our array
2604 for k
in marshalled
.keys():
2606 if 'streamContentSize' not in self
.stream_file
:
2607 self
.stream_file
['streamContentSize'] = 0
2608 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2609 self
.stream_contents
.append(marshalled
['data'])
2611 self
.stream_file
[k
] = marshalled
[k
]
2614 'streamContentSize' in self
.stream_file
and
2615 'fileSize' in self
.stream_file
and
2616 'depotFile' in self
.stream_file
):
2617 size
= int(self
.stream_file
["fileSize"])
2619 progress
= 100*self
.stream_file
['streamContentSize']/size
2620 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2623 self
.stream_have_file_info
= True
2625 # Stream directly from "p4 files" into "git fast-import"
2626 def streamP4Files(self
, files
):
2632 filesForCommit
.append(f
)
2633 if f
['action'] in self
.delete_actions
:
2634 filesToDelete
.append(f
)
2636 filesToRead
.append(f
)
2639 for f
in filesToDelete
:
2640 self
.streamOneP4Deletion(f
)
2642 if len(filesToRead
) > 0:
2643 self
.stream_file
= {}
2644 self
.stream_contents
= []
2645 self
.stream_have_file_info
= False
2647 # curry self argument
2648 def streamP4FilesCbSelf(entry
):
2649 self
.streamP4FilesCb(entry
)
2651 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2653 p4CmdList(["-x", "-", "print"],
2655 cb
=streamP4FilesCbSelf
)
2658 if self
.stream_file
.has_key('depotFile'):
2659 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2661 def make_email(self
, userid
):
2662 if userid
in self
.users
:
2663 return self
.users
[userid
]
2665 return "%s <a@b>" % userid
2667 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2668 """ Stream a p4 tag.
2669 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2673 print "writing tag %s for commit %s" % (labelName
, commit
)
2674 gitStream
.write("tag %s\n" % labelName
)
2675 gitStream
.write("from %s\n" % commit
)
2677 if labelDetails
.has_key('Owner'):
2678 owner
= labelDetails
["Owner"]
2682 # Try to use the owner of the p4 label, or failing that,
2683 # the current p4 user id.
2685 email
= self
.make_email(owner
)
2687 email
= self
.make_email(self
.p4UserId())
2688 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2690 gitStream
.write("tagger %s\n" % tagger
)
2692 print "labelDetails=",labelDetails
2693 if labelDetails
.has_key('Description'):
2694 description
= labelDetails
['Description']
2696 description
= 'Label from git p4'
2698 gitStream
.write("data %d\n" % len(description
))
2699 gitStream
.write(description
)
2700 gitStream
.write("\n")
2702 def inClientSpec(self
, path
):
2703 if not self
.clientSpecDirs
:
2705 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
2706 if not inClientSpec
and self
.verbose
:
2707 print('Ignoring file outside of client spec: {0}'.format(path
))
2710 def hasBranchPrefix(self
, path
):
2711 if not self
.branchPrefixes
:
2713 hasPrefix
= [p
for p
in self
.branchPrefixes
2714 if p4PathStartsWith(path
, p
)]
2715 if not hasPrefix
and self
.verbose
:
2716 print('Ignoring file outside of prefix: {0}'.format(path
))
2719 def commit(self
, details
, files
, branch
, parent
= ""):
2720 epoch
= details
["time"]
2721 author
= details
["user"]
2722 jobs
= self
.extractJobsFromCommit(details
)
2725 print('commit into {0}'.format(branch
))
2727 if self
.clientSpecDirs
:
2728 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2730 files
= [f
for f
in files
2731 if self
.inClientSpec(f
['path']) and self
.hasBranchPrefix(f
['path'])]
2733 if not files
and not gitConfigBool('git-p4.keepEmptyCommits'):
2734 print('Ignoring revision {0} as it would produce an empty commit.'
2735 .format(details
['change']))
2738 self
.gitStream
.write("commit %s\n" % branch
)
2739 self
.gitStream
.write("mark :%s\n" % details
["change"])
2740 self
.committedChanges
.add(int(details
["change"]))
2742 if author
not in self
.users
:
2743 self
.getUserMapFromPerforceServer()
2744 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2746 self
.gitStream
.write("committer %s\n" % committer
)
2748 self
.gitStream
.write("data <<EOT\n")
2749 self
.gitStream
.write(details
["desc"])
2751 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
2752 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2753 (','.join(self
.branchPrefixes
), details
["change"]))
2754 if len(details
['options']) > 0:
2755 self
.gitStream
.write(": options = %s" % details
['options'])
2756 self
.gitStream
.write("]\nEOT\n\n")
2760 print "parent %s" % parent
2761 self
.gitStream
.write("from %s\n" % parent
)
2763 self
.streamP4Files(files
)
2764 self
.gitStream
.write("\n")
2766 change
= int(details
["change"])
2768 if self
.labels
.has_key(change
):
2769 label
= self
.labels
[change
]
2770 labelDetails
= label
[0]
2771 labelRevisions
= label
[1]
2773 print "Change %s is labelled %s" % (change
, labelDetails
)
2775 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2776 for p
in self
.branchPrefixes
])
2778 if len(files
) == len(labelRevisions
):
2782 if info
["action"] in self
.delete_actions
:
2784 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2786 if cleanedFiles
== labelRevisions
:
2787 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2791 print ("Tag %s does not match with change %s: files do not match."
2792 % (labelDetails
["label"], change
))
2796 print ("Tag %s does not match with change %s: file count is different."
2797 % (labelDetails
["label"], change
))
2799 # Build a dictionary of changelists and labels, for "detect-labels" option.
2800 def getLabels(self
):
2803 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2804 if len(l
) > 0 and not self
.silent
:
2805 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2808 label
= output
["label"]
2812 print "Querying files for label %s" % label
2813 for file in p4CmdList(["files"] +
2814 ["%s...@%s" % (p
, label
)
2815 for p
in self
.depotPaths
]):
2816 revisions
[file["depotFile"]] = file["rev"]
2817 change
= int(file["change"])
2818 if change
> newestChange
:
2819 newestChange
= change
2821 self
.labels
[newestChange
] = [output
, revisions
]
2824 print "Label changes: %s" % self
.labels
.keys()
2826 # Import p4 labels as git tags. A direct mapping does not
2827 # exist, so assume that if all the files are at the same revision
2828 # then we can use that, or it's something more complicated we should
2830 def importP4Labels(self
, stream
, p4Labels
):
2832 print "import p4 labels: " + ' '.join(p4Labels
)
2834 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2835 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2836 if len(validLabelRegexp
) == 0:
2837 validLabelRegexp
= defaultLabelRegexp
2838 m
= re
.compile(validLabelRegexp
)
2840 for name
in p4Labels
:
2843 if not m
.match(name
):
2845 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2848 if name
in ignoredP4Labels
:
2851 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2853 # get the most recent changelist for each file in this label
2854 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2855 for p
in self
.depotPaths
])
2857 if change
.has_key('change'):
2858 # find the corresponding git commit; take the oldest commit
2859 changelist
= int(change
['change'])
2860 if changelist
in self
.committedChanges
:
2861 gitCommit
= ":%d" % changelist
# use a fast-import mark
2864 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2865 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
2866 if len(gitCommit
) == 0:
2867 print "importing label %s: could not find git commit for changelist %d" % (name
, changelist
)
2870 gitCommit
= gitCommit
.strip()
2873 # Convert from p4 time format
2875 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2877 print "Could not convert label time %s" % labelDetails
['Update']
2880 when
= int(time
.mktime(tmwhen
))
2881 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2883 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2886 print "Label %s has no changelists - possibly deleted?" % name
2889 # We can't import this label; don't try again as it will get very
2890 # expensive repeatedly fetching all the files for labels that will
2891 # never be imported. If the label is moved in the future, the
2892 # ignore will need to be removed manually.
2893 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2895 def guessProjectName(self
):
2896 for p
in self
.depotPaths
:
2899 p
= p
[p
.strip().rfind("/") + 1:]
2900 if not p
.endswith("/"):
2904 def getBranchMapping(self
):
2905 lostAndFoundBranches
= set()
2907 user
= gitConfig("git-p4.branchUser")
2909 command
= "branches -u %s" % user
2911 command
= "branches"
2913 for info
in p4CmdList(command
):
2914 details
= p4Cmd(["branch", "-o", info
["branch"]])
2916 while details
.has_key("View%s" % viewIdx
):
2917 paths
= details
["View%s" % viewIdx
].split(" ")
2918 viewIdx
= viewIdx
+ 1
2919 # require standard //depot/foo/... //depot/bar/... mapping
2920 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2923 destination
= paths
[1]
2925 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2926 source
= source
[len(self
.depotPaths
[0]):-4]
2927 destination
= destination
[len(self
.depotPaths
[0]):-4]
2929 if destination
in self
.knownBranches
:
2931 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2932 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2935 self
.knownBranches
[destination
] = source
2937 lostAndFoundBranches
.discard(destination
)
2939 if source
not in self
.knownBranches
:
2940 lostAndFoundBranches
.add(source
)
2942 # Perforce does not strictly require branches to be defined, so we also
2943 # check git config for a branch list.
2945 # Example of branch definition in git config file:
2947 # branchList=main:branchA
2948 # branchList=main:branchB
2949 # branchList=branchA:branchC
2950 configBranches
= gitConfigList("git-p4.branchList")
2951 for branch
in configBranches
:
2953 (source
, destination
) = branch
.split(":")
2954 self
.knownBranches
[destination
] = source
2956 lostAndFoundBranches
.discard(destination
)
2958 if source
not in self
.knownBranches
:
2959 lostAndFoundBranches
.add(source
)
2962 for branch
in lostAndFoundBranches
:
2963 self
.knownBranches
[branch
] = branch
2965 def getBranchMappingFromGitBranches(self
):
2966 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2967 for branch
in branches
.keys():
2968 if branch
== "master":
2971 branch
= branch
[len(self
.projectName
):]
2972 self
.knownBranches
[branch
] = branch
2974 def updateOptionDict(self
, d
):
2976 if self
.keepRepoPath
:
2977 option_keys
['keepRepoPath'] = 1
2979 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2981 def readOptions(self
, d
):
2982 self
.keepRepoPath
= (d
.has_key('options')
2983 and ('keepRepoPath' in d
['options']))
2985 def gitRefForBranch(self
, branch
):
2986 if branch
== "main":
2987 return self
.refPrefix
+ "master"
2989 if len(branch
) <= 0:
2992 return self
.refPrefix
+ self
.projectName
+ branch
2994 def gitCommitByP4Change(self
, ref
, change
):
2996 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2999 latestCommit
= parseRevision(ref
)
3003 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
3004 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
3009 log
= extractLogMessageFromGitCommit(next
)
3010 settings
= extractSettingsGitLog(log
)
3011 currentChange
= int(settings
['change'])
3013 print "current change %s" % currentChange
3015 if currentChange
== change
:
3017 print "found %s" % next
3020 if currentChange
< change
:
3021 earliestCommit
= "^%s" % next
3023 latestCommit
= "%s" % next
3027 def importNewBranch(self
, branch
, maxChange
):
3028 # make fast-import flush all changes to disk and update the refs using the checkpoint
3029 # command so that we can try to find the branch parent in the git history
3030 self
.gitStream
.write("checkpoint\n\n");
3031 self
.gitStream
.flush();
3032 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3033 range = "@1,%s" % maxChange
3034 #print "prefix" + branchPrefix
3035 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3036 if len(changes
) <= 0:
3038 firstChange
= changes
[0]
3039 #print "first change in branch: %s" % firstChange
3040 sourceBranch
= self
.knownBranches
[branch
]
3041 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3042 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3043 #print "source " + sourceBranch
3045 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3046 #print "branch parent: %s" % branchParentChange
3047 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3048 if len(gitParent
) > 0:
3049 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3050 #print "parent git commit: %s" % gitParent
3052 self
.importChanges(changes
)
3055 def searchParent(self
, parent
, branch
, target
):
3057 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
3058 "--no-merges", parent
]):
3060 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
3063 print "Found parent of %s in commit %s" % (branch
, blob
)
3070 def importChanges(self
, changes
):
3072 for change
in changes
:
3073 description
= p4_describe(change
)
3074 self
.updateOptionDict(description
)
3077 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3082 if self
.detectBranches
:
3083 branches
= self
.splitFilesIntoBranches(description
)
3084 for branch
in branches
.keys():
3086 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3087 self
.branchPrefixes
= [ branchPrefix
]
3091 filesForCommit
= branches
[branch
]
3094 print "branch is %s" % branch
3096 self
.updatedBranches
.add(branch
)
3098 if branch
not in self
.createdBranches
:
3099 self
.createdBranches
.add(branch
)
3100 parent
= self
.knownBranches
[branch
]
3101 if parent
== branch
:
3104 fullBranch
= self
.projectName
+ branch
3105 if fullBranch
not in self
.p4BranchesInGit
:
3107 print("\n Importing new branch %s" % fullBranch
);
3108 if self
.importNewBranch(branch
, change
- 1):
3110 self
.p4BranchesInGit
.append(fullBranch
)
3112 print("\n Resuming with change %s" % change
);
3115 print "parent determined through known branches: %s" % parent
3117 branch
= self
.gitRefForBranch(branch
)
3118 parent
= self
.gitRefForBranch(parent
)
3121 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
3123 if len(parent
) == 0 and branch
in self
.initialParents
:
3124 parent
= self
.initialParents
[branch
]
3125 del self
.initialParents
[branch
]
3129 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3131 print "Creating temporary branch: " + tempBranch
3132 self
.commit(description
, filesForCommit
, tempBranch
)
3133 self
.tempBranches
.append(tempBranch
)
3135 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3137 self
.commit(description
, filesForCommit
, branch
, blob
)
3140 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3141 self
.commit(description
, filesForCommit
, branch
, parent
)
3143 files
= self
.extractFilesFromCommit(description
)
3144 self
.commit(description
, files
, self
.branch
,
3146 # only needed once, to connect to the previous commit
3147 self
.initialParent
= ""
3149 print self
.gitError
.read()
3152 def importHeadRevision(self
, revision
):
3153 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3156 details
["user"] = "git perforce import user"
3157 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3158 % (' '.join(self
.depotPaths
), revision
))
3159 details
["change"] = revision
3163 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3165 for info
in p4CmdList(["files"] + fileArgs
):
3167 if 'code' in info
and info
['code'] == 'error':
3168 sys
.stderr
.write("p4 returned an error: %s\n"
3170 if info
['data'].find("must refer to client") >= 0:
3171 sys
.stderr
.write("This particular p4 error is misleading.\n")
3172 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3173 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3175 if 'p4ExitCode' in info
:
3176 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3180 change
= int(info
["change"])
3181 if change
> newestRevision
:
3182 newestRevision
= change
3184 if info
["action"] in self
.delete_actions
:
3185 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3186 #fileCnt = fileCnt + 1
3189 for prop
in ["depotFile", "rev", "action", "type" ]:
3190 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3192 fileCnt
= fileCnt
+ 1
3194 details
["change"] = newestRevision
3196 # Use time from top-most change so that all git p4 clones of
3197 # the same p4 repo have the same commit SHA1s.
3198 res
= p4_describe(newestRevision
)
3199 details
["time"] = res
["time"]
3201 self
.updateOptionDict(details
)
3203 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3205 print "IO error with git fast-import. Is your git version recent enough?"
3206 print self
.gitError
.read()
3209 def run(self
, args
):
3210 self
.depotPaths
= []
3211 self
.changeRange
= ""
3212 self
.previousDepotPaths
= []
3213 self
.hasOrigin
= False
3215 # map from branch depot path to parent branch
3216 self
.knownBranches
= {}
3217 self
.initialParents
= {}
3219 if self
.importIntoRemotes
:
3220 self
.refPrefix
= "refs/remotes/p4/"
3222 self
.refPrefix
= "refs/heads/p4/"
3224 if self
.syncWithOrigin
:
3225 self
.hasOrigin
= originP4BranchesExist()
3228 print 'Syncing with origin first, using "git fetch origin"'
3229 system("git fetch origin")
3231 branch_arg_given
= bool(self
.branch
)
3232 if len(self
.branch
) == 0:
3233 self
.branch
= self
.refPrefix
+ "master"
3234 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3235 system("git update-ref %s refs/heads/p4" % self
.branch
)
3236 system("git branch -D p4")
3238 # accept either the command-line option, or the configuration variable
3239 if self
.useClientSpec
:
3240 # will use this after clone to set the variable
3241 self
.useClientSpec_from_options
= True
3243 if gitConfigBool("git-p4.useclientspec"):
3244 self
.useClientSpec
= True
3245 if self
.useClientSpec
:
3246 self
.clientSpecDirs
= getClientSpec()
3248 # TODO: should always look at previous commits,
3249 # merge with previous imports, if possible.
3252 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3254 # branches holds mapping from branch name to sha1
3255 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3257 # restrict to just this one, disabling detect-branches
3258 if branch_arg_given
:
3259 short
= self
.branch
.split("/")[-1]
3260 if short
in branches
:
3261 self
.p4BranchesInGit
= [ short
]
3263 self
.p4BranchesInGit
= branches
.keys()
3265 if len(self
.p4BranchesInGit
) > 1:
3267 print "Importing from/into multiple branches"
3268 self
.detectBranches
= True
3269 for branch
in branches
.keys():
3270 self
.initialParents
[self
.refPrefix
+ branch
] = \
3274 print "branches: %s" % self
.p4BranchesInGit
3277 for branch
in self
.p4BranchesInGit
:
3278 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3280 settings
= extractSettingsGitLog(logMsg
)
3282 self
.readOptions(settings
)
3283 if (settings
.has_key('depot-paths')
3284 and settings
.has_key ('change')):
3285 change
= int(settings
['change']) + 1
3286 p4Change
= max(p4Change
, change
)
3288 depotPaths
= sorted(settings
['depot-paths'])
3289 if self
.previousDepotPaths
== []:
3290 self
.previousDepotPaths
= depotPaths
3293 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3294 prev_list
= prev
.split("/")
3295 cur_list
= cur
.split("/")
3296 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3297 if cur_list
[i
] <> prev_list
[i
]:
3301 paths
.append ("/".join(cur_list
[:i
+ 1]))
3303 self
.previousDepotPaths
= paths
3306 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3307 self
.changeRange
= "@%s,#head" % p4Change
3308 if not self
.silent
and not self
.detectBranches
:
3309 print "Performing incremental import into %s git branch" % self
.branch
3311 # accept multiple ref name abbreviations:
3312 # refs/foo/bar/branch -> use it exactly
3313 # p4/branch -> prepend refs/remotes/ or refs/heads/
3314 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3315 if not self
.branch
.startswith("refs/"):
3316 if self
.importIntoRemotes
:
3317 prepend
= "refs/remotes/"
3319 prepend
= "refs/heads/"
3320 if not self
.branch
.startswith("p4/"):
3322 self
.branch
= prepend
+ self
.branch
3324 if len(args
) == 0 and self
.depotPaths
:
3326 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3328 if self
.depotPaths
and self
.depotPaths
!= args
:
3329 print ("previous import used depot path %s and now %s was specified. "
3330 "This doesn't work!" % (' '.join (self
.depotPaths
),
3334 self
.depotPaths
= sorted(args
)
3339 # Make sure no revision specifiers are used when --changesfile
3341 bad_changesfile
= False
3342 if len(self
.changesFile
) > 0:
3343 for p
in self
.depotPaths
:
3344 if p
.find("@") >= 0 or p
.find("#") >= 0:
3345 bad_changesfile
= True
3348 die("Option --changesfile is incompatible with revision specifiers")
3351 for p
in self
.depotPaths
:
3352 if p
.find("@") != -1:
3353 atIdx
= p
.index("@")
3354 self
.changeRange
= p
[atIdx
:]
3355 if self
.changeRange
== "@all":
3356 self
.changeRange
= ""
3357 elif ',' not in self
.changeRange
:
3358 revision
= self
.changeRange
3359 self
.changeRange
= ""
3361 elif p
.find("#") != -1:
3362 hashIdx
= p
.index("#")
3363 revision
= p
[hashIdx
:]
3365 elif self
.previousDepotPaths
== []:
3366 # pay attention to changesfile, if given, else import
3367 # the entire p4 tree at the head revision
3368 if len(self
.changesFile
) == 0:
3371 p
= re
.sub ("\.\.\.$", "", p
)
3372 if not p
.endswith("/"):
3377 self
.depotPaths
= newPaths
3379 # --detect-branches may change this for each branch
3380 self
.branchPrefixes
= self
.depotPaths
3382 self
.loadUserMapFromCache()
3384 if self
.detectLabels
:
3387 if self
.detectBranches
:
3388 ## FIXME - what's a P4 projectName ?
3389 self
.projectName
= self
.guessProjectName()
3392 self
.getBranchMappingFromGitBranches()
3394 self
.getBranchMapping()
3396 print "p4-git branches: %s" % self
.p4BranchesInGit
3397 print "initial parents: %s" % self
.initialParents
3398 for b
in self
.p4BranchesInGit
:
3402 b
= b
[len(self
.projectName
):]
3403 self
.createdBranches
.add(b
)
3405 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
3407 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3408 stdin
=subprocess
.PIPE
,
3409 stdout
=subprocess
.PIPE
,
3410 stderr
=subprocess
.PIPE
);
3411 self
.gitOutput
= self
.importProcess
.stdout
3412 self
.gitStream
= self
.importProcess
.stdin
3413 self
.gitError
= self
.importProcess
.stderr
3416 self
.importHeadRevision(revision
)
3420 if len(self
.changesFile
) > 0:
3421 output
= open(self
.changesFile
).readlines()
3424 changeSet
.add(int(line
))
3426 for change
in changeSet
:
3427 changes
.append(change
)
3431 # catch "git p4 sync" with no new branches, in a repo that
3432 # does not have any existing p4 branches
3434 if not self
.p4BranchesInGit
:
3435 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3437 # The default branch is master, unless --branch is used to
3438 # specify something else. Make sure it exists, or complain
3439 # nicely about how to use --branch.
3440 if not self
.detectBranches
:
3441 if not branch_exists(self
.branch
):
3442 if branch_arg_given
:
3443 die("Error: branch %s does not exist." % self
.branch
)
3445 die("Error: no branch %s; perhaps specify one with --branch." %
3449 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3451 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3453 if len(self
.maxChanges
) > 0:
3454 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3456 if len(changes
) == 0:
3458 print "No changes to import!"
3460 if not self
.silent
and not self
.detectBranches
:
3461 print "Import destination: %s" % self
.branch
3463 self
.updatedBranches
= set()
3465 if not self
.detectBranches
:
3467 # start a new branch
3468 self
.initialParent
= ""
3470 # build on a previous revision
3471 self
.initialParent
= parseRevision(self
.branch
)
3473 self
.importChanges(changes
)
3477 if len(self
.updatedBranches
) > 0:
3478 sys
.stdout
.write("Updated branches: ")
3479 for b
in self
.updatedBranches
:
3480 sys
.stdout
.write("%s " % b
)
3481 sys
.stdout
.write("\n")
3483 if gitConfigBool("git-p4.importLabels"):
3484 self
.importLabels
= True
3486 if self
.importLabels
:
3487 p4Labels
= getP4Labels(self
.depotPaths
)
3488 gitTags
= getGitTags()
3490 missingP4Labels
= p4Labels
- gitTags
3491 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3493 self
.gitStream
.close()
3494 if self
.importProcess
.wait() != 0:
3495 die("fast-import failed: %s" % self
.gitError
.read())
3496 self
.gitOutput
.close()
3497 self
.gitError
.close()
3499 # Cleanup temporary branches created during import
3500 if self
.tempBranches
!= []:
3501 for branch
in self
.tempBranches
:
3502 read_pipe("git update-ref -d %s" % branch
)
3503 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3505 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3506 # a convenient shortcut refname "p4".
3507 if self
.importIntoRemotes
:
3508 head_ref
= self
.refPrefix
+ "HEAD"
3509 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3510 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3514 class P4Rebase(Command
):
3516 Command
.__init
__(self
)
3518 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3520 self
.importLabels
= False
3521 self
.description
= ("Fetches the latest revision from perforce and "
3522 + "rebases the current work (branch) against it")
3524 def run(self
, args
):
3526 sync
.importLabels
= self
.importLabels
3529 return self
.rebase()
3532 if os
.system("git update-index --refresh") != 0:
3533 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3534 if len(read_pipe("git diff-index HEAD --")) > 0:
3535 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3537 [upstream
, settings
] = findUpstreamBranchPoint()
3538 if len(upstream
) == 0:
3539 die("Cannot find upstream branchpoint for rebase")
3541 # the branchpoint may be p4/foo~3, so strip off the parent
3542 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3544 print "Rebasing the current branch onto %s" % upstream
3545 oldHead
= read_pipe("git rev-parse HEAD").strip()
3546 system("git rebase %s" % upstream
)
3547 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3550 class P4Clone(P4Sync
):
3552 P4Sync
.__init
__(self
)
3553 self
.description
= "Creates a new git repository and imports from Perforce into it"
3554 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3556 optparse
.make_option("--destination", dest
="cloneDestination",
3557 action
='store', default
=None,
3558 help="where to leave result of the clone"),
3559 optparse
.make_option("--bare", dest
="cloneBare",
3560 action
="store_true", default
=False),
3562 self
.cloneDestination
= None
3563 self
.needsGit
= False
3564 self
.cloneBare
= False
3566 def defaultDestination(self
, args
):
3567 ## TODO: use common prefix of args?
3569 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3570 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3571 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3572 depotDir
= re
.sub(r
"/$", "", depotDir
)
3573 return os
.path
.split(depotDir
)[1]
3575 def run(self
, args
):
3579 if self
.keepRepoPath
and not self
.cloneDestination
:
3580 sys
.stderr
.write("Must specify destination for --keep-path\n")
3585 if not self
.cloneDestination
and len(depotPaths
) > 1:
3586 self
.cloneDestination
= depotPaths
[-1]
3587 depotPaths
= depotPaths
[:-1]
3589 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3590 for p
in depotPaths
:
3591 if not p
.startswith("//"):
3592 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3595 if not self
.cloneDestination
:
3596 self
.cloneDestination
= self
.defaultDestination(args
)
3598 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3600 if not os
.path
.exists(self
.cloneDestination
):
3601 os
.makedirs(self
.cloneDestination
)
3602 chdir(self
.cloneDestination
)
3604 init_cmd
= [ "git", "init" ]
3606 init_cmd
.append("--bare")
3607 retcode
= subprocess
.call(init_cmd
)
3609 raise CalledProcessError(retcode
, init_cmd
)
3611 if not P4Sync
.run(self
, depotPaths
):
3614 # create a master branch and check out a work tree
3615 if gitBranchExists(self
.branch
):
3616 system([ "git", "branch", "master", self
.branch
])
3617 if not self
.cloneBare
:
3618 system([ "git", "checkout", "-f" ])
3620 print 'Not checking out any branch, use ' \
3621 '"git checkout -q -b master <branch>"'
3623 # auto-set this variable if invoked with --use-client-spec
3624 if self
.useClientSpec_from_options
:
3625 system("git config --bool git-p4.useclientspec true")
3629 class P4Branches(Command
):
3631 Command
.__init
__(self
)
3633 self
.description
= ("Shows the git branches that hold imports and their "
3634 + "corresponding perforce depot paths")
3635 self
.verbose
= False
3637 def run(self
, args
):
3638 if originP4BranchesExist():
3639 createOrUpdateBranchesFromOrigin()
3641 cmdline
= "git rev-parse --symbolic "
3642 cmdline
+= " --remotes"
3644 for line
in read_pipe_lines(cmdline
):
3647 if not line
.startswith('p4/') or line
== "p4/HEAD":
3651 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3652 settings
= extractSettingsGitLog(log
)
3654 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3657 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3659 optparse
.IndentedHelpFormatter
.__init
__(self
)
3661 def format_description(self
, description
):
3663 return description
+ "\n"
3667 def printUsage(commands
):
3668 print "usage: %s <command> [options]" % sys
.argv
[0]
3670 print "valid commands: %s" % ", ".join(commands
)
3672 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3677 "submit" : P4Submit
,
3678 "commit" : P4Submit
,
3680 "rebase" : P4Rebase
,
3682 "rollback" : P4RollBack
,
3683 "branches" : P4Branches
3688 if len(sys
.argv
[1:]) == 0:
3689 printUsage(commands
.keys())
3692 cmdName
= sys
.argv
[1]
3694 klass
= commands
[cmdName
]
3697 print "unknown command %s" % cmdName
3699 printUsage(commands
.keys())
3702 options
= cmd
.options
3703 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3707 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3709 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3711 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3713 description
= cmd
.description
,
3714 formatter
= HelpFormatter())
3716 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3718 verbose
= cmd
.verbose
3720 if cmd
.gitdir
== None:
3721 cmd
.gitdir
= os
.path
.abspath(".git")
3722 if not isValidGitDir(cmd
.gitdir
):
3723 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3724 if os
.path
.exists(cmd
.gitdir
):
3725 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3729 if not isValidGitDir(cmd
.gitdir
):
3730 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3731 cmd
.gitdir
+= "/.git"
3733 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3735 os
.environ
["GIT_DIR"] = cmd
.gitdir
3737 if not cmd
.run(args
):
3742 if __name__
== '__main__':