3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess
import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self
, returncode
, cmd
):
39 self
.returncode
= returncode
42 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize
= 512
52 def p4_build_cmd(cmd
):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user
= gitConfig("git-p4.user")
63 real_cmd
+= ["-u",user
]
65 password
= gitConfig("git-p4.password")
67 real_cmd
+= ["-P", password
]
69 port
= gitConfig("git-p4.port")
71 real_cmd
+= ["-p", port
]
73 host
= gitConfig("git-p4.host")
75 real_cmd
+= ["-H", host
]
77 client
= gitConfig("git-p4.client")
79 real_cmd
+= ["-c", client
]
82 if isinstance(cmd
,basestring
):
83 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
88 def chdir(path
, is_client_path
=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
106 if not is_client_path
:
108 os
.environ
['PWD'] = path
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform
.system() == 'Windows':
113 free_bytes
= ctypes
.c_ulonglong(0)
114 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
115 return free_bytes
.value
117 st
= os
.statvfs(os
.getcwd())
118 return st
.f_bavail
* st
.f_frsize
124 sys
.stderr
.write(msg
+ "\n")
127 def write_pipe(c
, stdin
):
129 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
131 expand
= isinstance(c
,basestring
)
132 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
134 val
= pipe
.write(stdin
)
137 die('Command failed: %s' % str(c
))
141 def p4_write_pipe(c
, stdin
):
142 real_cmd
= p4_build_cmd(c
)
143 return write_pipe(real_cmd
, stdin
)
145 def read_pipe(c
, ignore_error
=False):
147 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
149 expand
= isinstance(c
,basestring
)
150 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
151 (out
, err
) = p
.communicate()
152 if p
.returncode
!= 0 and not ignore_error
:
153 die('Command failed: %s\nError: %s' % (str(c
), err
))
156 def p4_read_pipe(c
, ignore_error
=False):
157 real_cmd
= p4_build_cmd(c
)
158 return read_pipe(real_cmd
, ignore_error
)
160 def read_pipe_lines(c
):
162 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
164 expand
= isinstance(c
, basestring
)
165 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
167 val
= pipe
.readlines()
168 if pipe
.close() or p
.wait():
169 die('Command failed: %s' % str(c
))
173 def p4_read_pipe_lines(c
):
174 """Specifically invoke p4 on the command supplied. """
175 real_cmd
= p4_build_cmd(c
)
176 return read_pipe_lines(real_cmd
)
178 def p4_has_command(cmd
):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd
= p4_build_cmd(["help", cmd
])
182 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
183 stderr
=subprocess
.PIPE
)
185 return p
.returncode
== 0
187 def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
193 if not p4_has_command("move"):
195 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
196 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
197 (out
, err
) = p
.communicate()
198 # return code will be 1 in either case
199 if err
.find("Invalid option") >= 0:
201 if err
.find("disabled") >= 0:
203 # assume it failed because @... was invalid changelist
206 def system(cmd
, ignore_error
=False):
207 expand
= isinstance(cmd
,basestring
)
209 sys
.stderr
.write("executing %s\n" % str(cmd
))
210 retcode
= subprocess
.call(cmd
, shell
=expand
)
211 if retcode
and not ignore_error
:
212 raise CalledProcessError(retcode
, cmd
)
217 """Specifically invoke p4 as the system command. """
218 real_cmd
= p4_build_cmd(cmd
)
219 expand
= isinstance(real_cmd
, basestring
)
220 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
222 raise CalledProcessError(retcode
, real_cmd
)
224 _p4_version_string
= None
225 def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
234 global _p4_version_string
235 if not _p4_version_string
:
236 a
= p4_read_pipe_lines(["-V"])
237 _p4_version_string
= a
[-1].rstrip()
238 return _p4_version_string
240 def p4_integrate(src
, dest
):
241 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
243 def p4_sync(f
, *options
):
244 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
247 # forcibly add file names with wildcards
248 if wildcard_present(f
):
249 p4_system(["add", "-f", f
])
251 p4_system(["add", f
])
254 p4_system(["delete", wildcard_encode(f
)])
257 p4_system(["edit", wildcard_encode(f
)])
260 p4_system(["revert", wildcard_encode(f
)])
262 def p4_reopen(type, f
):
263 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
265 def p4_move(src
, dest
):
266 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
268 def p4_last_change():
269 results
= p4CmdList(["changes", "-m", "1"])
270 return int(results
[0]['change'])
272 def p4_describe(change
):
273 """Make sure it returns a valid result by checking for
274 the presence of field "time". Return a dict of the
277 ds
= p4CmdList(["describe", "-s", str(change
)])
279 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
283 if "p4ExitCode" in d
:
284 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
287 if d
["code"] == "error":
288 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
291 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
296 # Canonicalize the p4 type and return a tuple of the
297 # base type, plus any modifiers. See "p4 help filetypes"
298 # for a list and explanation.
300 def split_p4_type(p4type
):
302 p4_filetypes_historical
= {
303 "ctempobj": "binary+Sw",
309 "tempobj": "binary+FSw",
310 "ubinary": "binary+F",
311 "uresource": "resource+F",
312 "uxbinary": "binary+Fx",
313 "xbinary": "binary+x",
315 "xtempobj": "binary+Swx",
317 "xunicode": "unicode+x",
320 if p4type
in p4_filetypes_historical
:
321 p4type
= p4_filetypes_historical
[p4type
]
323 s
= p4type
.split("+")
331 # return the raw p4 type of a file (text, text+ko, etc)
334 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
335 return results
[0]['headType']
338 # Given a type base and modifier, return a regexp matching
339 # the keywords that can be expanded in the file
341 def p4_keywords_regexp_for_type(base
, type_mods
):
342 if base
in ("text", "unicode", "binary"):
344 if "ko" in type_mods
:
346 elif "k" in type_mods
:
347 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
351 \$ # Starts with a dollar, followed by...
352 (%s) # one of the keywords, followed by...
353 (:[^$\n]+)? # possibly an old expansion, followed by...
361 # Given a file, return a regexp matching the possible
362 # RCS keywords that will be expanded, or None for files
363 # with kw expansion turned off.
365 def p4_keywords_regexp_for_file(file):
366 if not os
.path
.exists(file):
369 (type_base
, type_mods
) = split_p4_type(p4_type(file))
370 return p4_keywords_regexp_for_type(type_base
, type_mods
)
372 def setP4ExecBit(file, mode
):
373 # Reopens an already open file and changes the execute bit to match
374 # the execute bit setting in the passed in mode.
378 if not isModeExec(mode
):
379 p4Type
= getP4OpenedType(file)
380 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
381 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
382 if p4Type
[-1] == "+":
383 p4Type
= p4Type
[0:-1]
385 p4_reopen(p4Type
, file)
387 def getP4OpenedType(file):
388 # Returns the perforce file type for the given file.
390 result
= p4_read_pipe(["opened", wildcard_encode(file)])
391 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
393 return match
.group(1)
395 die("Could not determine file type for %s (result: '%s')" % (file, result
))
397 # Return the set of all p4 labels
398 def getP4Labels(depotPaths
):
400 if isinstance(depotPaths
,basestring
):
401 depotPaths
= [depotPaths
]
403 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
409 # Return the set of all git tags
412 for line
in read_pipe_lines(["git", "tag"]):
417 def diffTreePattern():
418 # This is a simple generator for the diff tree regex pattern. This could be
419 # a class variable if this and parseDiffTreeEntry were a part of a class.
420 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
424 def parseDiffTreeEntry(entry
):
425 """Parses a single diff tree entry into its component elements.
427 See git-diff-tree(1) manpage for details about the format of the diff
428 output. This method returns a dictionary with the following elements:
430 src_mode - The mode of the source file
431 dst_mode - The mode of the destination file
432 src_sha1 - The sha1 for the source file
433 dst_sha1 - The sha1 fr the destination file
434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435 status_score - The score for the status (applicable for 'C' and 'R'
436 statuses). This is None if there is no score.
437 src - The path for the source file.
438 dst - The path for the destination file. This is only present for
439 copy or renames. If it is not present, this is None.
441 If the pattern is not matched, None is returned."""
443 match
= diffTreePattern().next().match(entry
)
446 'src_mode': match
.group(1),
447 'dst_mode': match
.group(2),
448 'src_sha1': match
.group(3),
449 'dst_sha1': match
.group(4),
450 'status': match
.group(5),
451 'status_score': match
.group(6),
452 'src': match
.group(7),
453 'dst': match
.group(10)
457 def isModeExec(mode
):
458 # Returns True if the given git mode represents an executable file,
460 return mode
[-3:] == "755"
462 def isModeExecChanged(src_mode
, dst_mode
):
463 return isModeExec(src_mode
) != isModeExec(dst_mode
)
465 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
467 if isinstance(cmd
,basestring
):
474 cmd
= p4_build_cmd(cmd
)
476 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
478 # Use a temporary file to avoid deadlocks without
479 # subprocess.communicate(), which would put another copy
480 # of stdout into memory.
482 if stdin
is not None:
483 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
484 if isinstance(stdin
,basestring
):
485 stdin_file
.write(stdin
)
488 stdin_file
.write(i
+ '\n')
492 p4
= subprocess
.Popen(cmd
,
495 stdout
=subprocess
.PIPE
)
500 entry
= marshal
.load(p4
.stdout
)
510 entry
["p4ExitCode"] = exitCode
516 list = p4CmdList(cmd
)
522 def p4Where(depotPath
):
523 if not depotPath
.endswith("/"):
525 depotPathLong
= depotPath
+ "..."
526 outputList
= p4CmdList(["where", depotPathLong
])
528 for entry
in outputList
:
529 if "depotFile" in entry
:
530 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531 # The base path always ends with "/...".
532 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
535 elif "data" in entry
:
536 data
= entry
.get("data")
537 space
= data
.find(" ")
538 if data
[:space
] == depotPath
:
543 if output
["code"] == "error":
547 clientPath
= output
.get("path")
548 elif "data" in output
:
549 data
= output
.get("data")
550 lastSpace
= data
.rfind(" ")
551 clientPath
= data
[lastSpace
+ 1:]
553 if clientPath
.endswith("..."):
554 clientPath
= clientPath
[:-3]
557 def currentGitBranch():
558 retcode
= system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error
=True)
563 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
565 def isValidGitDir(path
):
566 if (os
.path
.exists(path
+ "/HEAD")
567 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
571 def parseRevision(ref
):
572 return read_pipe("git rev-parse %s" % ref
).strip()
574 def branchExists(ref
):
575 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
579 def extractLogMessageFromGitCommit(commit
):
582 ## fixme: title is first line of commit, not 1st paragraph.
584 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
593 def extractSettingsGitLog(log
):
595 for line
in log
.split("\n"):
597 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
601 assignments
= m
.group(1).split (':')
602 for a
in assignments
:
604 key
= vals
[0].strip()
605 val
= ('='.join (vals
[1:])).strip()
606 if val
.endswith ('\"') and val
.startswith('"'):
611 paths
= values
.get("depot-paths")
613 paths
= values
.get("depot-path")
615 values
['depot-paths'] = paths
.split(',')
618 def gitBranchExists(branch
):
619 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
620 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
621 return proc
.wait() == 0;
625 def gitConfig(key
, typeSpecifier
=None):
626 if not _gitConfig
.has_key(key
):
627 cmd
= [ "git", "config" ]
629 cmd
+= [ typeSpecifier
]
631 s
= read_pipe(cmd
, ignore_error
=True)
632 _gitConfig
[key
] = s
.strip()
633 return _gitConfig
[key
]
635 def gitConfigBool(key
):
636 """Return a bool, using git config --bool. It is True only if the
637 variable is set to true, and False if set to false or not present
640 if not _gitConfig
.has_key(key
):
641 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
642 return _gitConfig
[key
]
644 def gitConfigInt(key
):
645 if not _gitConfig
.has_key(key
):
646 cmd
= [ "git", "config", "--int", key
]
647 s
= read_pipe(cmd
, ignore_error
=True)
650 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
652 _gitConfig
[key
] = None
653 return _gitConfig
[key
]
655 def gitConfigList(key
):
656 if not _gitConfig
.has_key(key
):
657 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
658 _gitConfig
[key
] = s
.strip().split(os
.linesep
)
659 if _gitConfig
[key
] == ['']:
661 return _gitConfig
[key
]
663 def p4BranchesInGit(branchesAreInRemotes
=True):
664 """Find all the branches whose names start with "p4/", looking
665 in remotes or heads as specified by the argument. Return
666 a dictionary of { branch: revision } for each one found.
667 The branch names are the short names, without any
672 cmdline
= "git rev-parse --symbolic "
673 if branchesAreInRemotes
:
674 cmdline
+= "--remotes"
676 cmdline
+= "--branches"
678 for line
in read_pipe_lines(cmdline
):
682 if not line
.startswith('p4/'):
684 # special symbolic ref to p4/master
685 if line
== "p4/HEAD":
688 # strip off p4/ prefix
689 branch
= line
[len("p4/"):]
691 branches
[branch
] = parseRevision(line
)
695 def branch_exists(branch
):
696 """Make sure that the given ref name really exists."""
698 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
699 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
700 out
, _
= p
.communicate()
703 # expect exactly one line of output: the branch name
704 return out
.rstrip() == branch
706 def findUpstreamBranchPoint(head
= "HEAD"):
707 branches
= p4BranchesInGit()
708 # map from depot-path to branch name
709 branchByDepotPath
= {}
710 for branch
in branches
.keys():
711 tip
= branches
[branch
]
712 log
= extractLogMessageFromGitCommit(tip
)
713 settings
= extractSettingsGitLog(log
)
714 if settings
.has_key("depot-paths"):
715 paths
= ",".join(settings
["depot-paths"])
716 branchByDepotPath
[paths
] = "remotes/p4/" + branch
720 while parent
< 65535:
721 commit
= head
+ "~%s" % parent
722 log
= extractLogMessageFromGitCommit(commit
)
723 settings
= extractSettingsGitLog(log
)
724 if settings
.has_key("depot-paths"):
725 paths
= ",".join(settings
["depot-paths"])
726 if branchByDepotPath
.has_key(paths
):
727 return [branchByDepotPath
[paths
], settings
]
731 return ["", settings
]
733 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
735 print ("Creating/updating branch(es) in %s based on origin branch(es)"
738 originPrefix
= "origin/p4/"
740 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
742 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
745 headName
= line
[len(originPrefix
):]
746 remoteHead
= localRefPrefix
+ headName
749 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
750 if (not original
.has_key('depot-paths')
751 or not original
.has_key('change')):
755 if not gitBranchExists(remoteHead
):
757 print "creating %s" % remoteHead
760 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
761 if settings
.has_key('change') > 0:
762 if settings
['depot-paths'] == original
['depot-paths']:
763 originP4Change
= int(original
['change'])
764 p4Change
= int(settings
['change'])
765 if originP4Change
> p4Change
:
766 print ("%s (%s) is newer than %s (%s). "
767 "Updating p4 branch from origin."
768 % (originHead
, originP4Change
,
769 remoteHead
, p4Change
))
772 print ("Ignoring: %s was imported from %s while "
773 "%s was imported from %s"
774 % (originHead
, ','.join(original
['depot-paths']),
775 remoteHead
, ','.join(settings
['depot-paths'])))
778 system("git update-ref %s %s" % (remoteHead
, originHead
))
780 def originP4BranchesExist():
781 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
784 def p4ParseNumericChangeRange(parts
):
785 changeStart
= int(parts
[0][1:])
786 if parts
[1] == '#head':
787 changeEnd
= p4_last_change()
789 changeEnd
= int(parts
[1])
791 return (changeStart
, changeEnd
)
793 def chooseBlockSize(blockSize
):
797 return defaultBlockSize
799 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
802 # Parse the change range into start and end. Try to find integer
803 # revision ranges as these can be broken up into blocks to avoid
804 # hitting server-side limits (maxrows, maxscanresults). But if
805 # that doesn't work, fall back to using the raw revision specifier
806 # strings, without using block mode.
808 if changeRange
is None or changeRange
== '':
810 changeEnd
= p4_last_change()
811 block_size
= chooseBlockSize(requestedBlockSize
)
813 parts
= changeRange
.split(',')
814 assert len(parts
) == 2
816 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
817 block_size
= chooseBlockSize(requestedBlockSize
)
819 changeStart
= parts
[0][1:]
821 if requestedBlockSize
:
822 die("cannot use --changes-block-size with non-numeric revisions")
825 # Accumulate change numbers in a dictionary to avoid duplicates
829 # Retrieve changes a block at a time, to prevent running
830 # into a MaxResults/MaxScanRows error from the server.
836 end
= min(changeEnd
, changeStart
+ block_size
)
837 revisionRange
= "%d,%d" % (changeStart
, end
)
839 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
841 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
843 for line
in p4_read_pipe_lines(cmd
):
844 changeNum
= int(line
.split(" ")[1])
845 changes
[changeNum
] = True
853 changeStart
= end
+ 1
855 changelist
= changes
.keys()
859 def p4PathStartsWith(path
, prefix
):
860 # This method tries to remedy a potential mixed-case issue:
862 # If UserA adds //depot/DirA/file1
863 # and UserB adds //depot/dira/file2
865 # we may or may not have a problem. If you have core.ignorecase=true,
866 # we treat DirA and dira as the same directory
867 if gitConfigBool("core.ignorecase"):
868 return path
.lower().startswith(prefix
.lower())
869 return path
.startswith(prefix
)
872 """Look at the p4 client spec, create a View() object that contains
873 all the mappings, and return it."""
875 specList
= p4CmdList("client -o")
876 if len(specList
) != 1:
877 die('Output from "client -o" is %d lines, expecting 1' %
880 # dictionary of all client parameters
884 client_name
= entry
["Client"]
886 # just the keys that start with "View"
887 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
890 view
= View(client_name
)
892 # append the lines, in order, to the view
893 for view_num
in range(len(view_keys
)):
894 k
= "View%d" % view_num
895 if k
not in view_keys
:
896 die("Expected view key %s missing" % k
)
897 view
.append(entry
[k
])
902 """Grab the client directory."""
904 output
= p4CmdList("client -o")
906 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
909 if "Root" not in entry
:
910 die('Client has no "Root"')
915 # P4 wildcards are not allowed in filenames. P4 complains
916 # if you simply add them, but you can force it with "-f", in
917 # which case it translates them into %xx encoding internally.
919 def wildcard_decode(path
):
920 # Search for and fix just these four characters. Do % last so
921 # that fixing it does not inadvertently create new %-escapes.
922 # Cannot have * in a filename in windows; untested as to
923 # what p4 would do in such a case.
924 if not platform
.system() == "Windows":
925 path
= path
.replace("%2A", "*")
926 path
= path
.replace("%23", "#") \
927 .replace("%40", "@") \
931 def wildcard_encode(path
):
932 # do % first to avoid double-encoding the %s introduced here
933 path
= path
.replace("%", "%25") \
934 .replace("*", "%2A") \
935 .replace("#", "%23") \
939 def wildcard_present(path
):
940 m
= re
.search("[*#@%]", path
)
943 class LargeFileSystem(object):
944 """Base class for large file system support."""
946 def __init__(self
, writeToGitStream
):
947 self
.largeFiles
= set()
948 self
.writeToGitStream
= writeToGitStream
950 def generatePointer(self
, cloneDestination
, contentFile
):
951 """Return the content of a pointer file that is stored in Git instead of
952 the actual content."""
953 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
955 def pushFile(self
, localLargeFile
):
956 """Push the actual content which is not stored in the Git repository to
958 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
960 def hasLargeFileExtension(self
, relPath
):
963 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
967 def generateTempFile(self
, contents
):
968 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
972 return contentFile
.name
974 def exceedsLargeFileThreshold(self
, relPath
, contents
):
975 if gitConfigInt('git-p4.largeFileThreshold'):
976 contentsSize
= sum(len(d
) for d
in contents
)
977 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
979 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
980 contentsSize
= sum(len(d
) for d
in contents
)
981 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
983 contentTempFile
= self
.generateTempFile(contents
)
984 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
985 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
986 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
988 compressedContentsSize
= zf
.infolist()[0].compress_size
989 os
.remove(contentTempFile
)
990 os
.remove(compressedContentFile
.name
)
991 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
995 def addLargeFile(self
, relPath
):
996 self
.largeFiles
.add(relPath
)
998 def removeLargeFile(self
, relPath
):
999 self
.largeFiles
.remove(relPath
)
1001 def isLargeFile(self
, relPath
):
1002 return relPath
in self
.largeFiles
1004 def processContent(self
, git_mode
, relPath
, contents
):
1005 """Processes the content of git fast import. This method decides if a
1006 file is stored in the large file system and handles all necessary
1008 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1009 contentTempFile
= self
.generateTempFile(contents
)
1010 (git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1012 # Move temp file to final location in large file system
1013 largeFileDir
= os
.path
.dirname(localLargeFile
)
1014 if not os
.path
.isdir(largeFileDir
):
1015 os
.makedirs(largeFileDir
)
1016 shutil
.move(contentTempFile
, localLargeFile
)
1017 self
.addLargeFile(relPath
)
1018 if gitConfigBool('git-p4.largeFilePush'):
1019 self
.pushFile(localLargeFile
)
1021 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1022 return (git_mode
, contents
)
1024 class MockLFS(LargeFileSystem
):
1025 """Mock large file system for testing."""
1027 def generatePointer(self
, contentFile
):
1028 """The pointer content is the original content prefixed with "pointer-".
1029 The local filename of the large file storage is derived from the file content.
1031 with
open(contentFile
, 'r') as f
:
1034 pointerContents
= 'pointer-' + content
1035 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1036 return (gitMode
, pointerContents
, localLargeFile
)
1038 def pushFile(self
, localLargeFile
):
1039 """The remote filename of the large file storage is the same as the local
1040 one but in a different directory.
1042 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1043 if not os
.path
.exists(remotePath
):
1044 os
.makedirs(remotePath
)
1045 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1047 class GitLFS(LargeFileSystem
):
1048 """Git LFS as backend for the git-p4 large file system.
1049 See https://git-lfs.github.com/ for details."""
1051 def __init__(self
, *args
):
1052 LargeFileSystem
.__init
__(self
, *args
)
1053 self
.baseGitAttributes
= []
1055 def generatePointer(self
, contentFile
):
1056 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1057 mode and content which is stored in the Git repository instead of
1058 the actual content. Return also the new location of the actual
1061 pointerProcess
= subprocess
.Popen(
1062 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1063 stdout
=subprocess
.PIPE
1065 pointerFile
= pointerProcess
.stdout
.read()
1066 if pointerProcess
.wait():
1067 os
.remove(contentFile
)
1068 die('git-lfs pointer command failed. Did you install the extension?')
1069 pointerContents
= [i
+'\n' for i
in pointerFile
.split('\n')[2:][:-1]]
1070 oid
= pointerContents
[1].split(' ')[1].split(':')[1][:-1]
1071 localLargeFile
= os
.path
.join(
1073 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1076 # LFS Spec states that pointer files should not have the executable bit set.
1078 return (gitMode
, pointerContents
, localLargeFile
)
1080 def pushFile(self
, localLargeFile
):
1081 uploadProcess
= subprocess
.Popen(
1082 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1084 if uploadProcess
.wait():
1085 die('git-lfs push command failed. Did you define a remote?')
1087 def generateGitAttributes(self
):
1089 self
.baseGitAttributes
+
1093 '# Git LFS (see https://git-lfs.github.com/)\n',
1096 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1097 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1099 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1100 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1104 def addLargeFile(self
, relPath
):
1105 LargeFileSystem
.addLargeFile(self
, relPath
)
1106 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1108 def removeLargeFile(self
, relPath
):
1109 LargeFileSystem
.removeLargeFile(self
, relPath
)
1110 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1112 def processContent(self
, git_mode
, relPath
, contents
):
1113 if relPath
== '.gitattributes':
1114 self
.baseGitAttributes
= contents
1115 return (git_mode
, self
.generateGitAttributes())
1117 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1121 self
.usage
= "usage: %prog [options]"
1122 self
.needsGit
= True
1123 self
.verbose
= False
1127 self
.userMapFromPerforceServer
= False
1128 self
.myP4UserId
= None
1132 return self
.myP4UserId
1134 results
= p4CmdList("user -o")
1136 if r
.has_key('User'):
1137 self
.myP4UserId
= r
['User']
1139 die("Could not find your p4 user id")
1141 def p4UserIsMe(self
, p4User
):
1142 # return True if the given p4 user is actually me
1143 me
= self
.p4UserId()
1144 if not p4User
or p4User
!= me
:
1149 def getUserCacheFilename(self
):
1150 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1151 return home
+ "/.gitp4-usercache.txt"
1153 def getUserMapFromPerforceServer(self
):
1154 if self
.userMapFromPerforceServer
:
1159 for output
in p4CmdList("users"):
1160 if not output
.has_key("User"):
1162 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1163 self
.emails
[output
["Email"]] = output
["User"]
1167 for (key
, val
) in self
.users
.items():
1168 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1170 open(self
.getUserCacheFilename(), "wb").write(s
)
1171 self
.userMapFromPerforceServer
= True
1173 def loadUserMapFromCache(self
):
1175 self
.userMapFromPerforceServer
= False
1177 cache
= open(self
.getUserCacheFilename(), "rb")
1178 lines
= cache
.readlines()
1181 entry
= line
.strip().split("\t")
1182 self
.users
[entry
[0]] = entry
[1]
1184 self
.getUserMapFromPerforceServer()
1186 class P4Debug(Command
):
1188 Command
.__init
__(self
)
1190 self
.description
= "A tool to debug the output of p4 -G."
1191 self
.needsGit
= False
1193 def run(self
, args
):
1195 for output
in p4CmdList(args
):
1196 print 'Element: %d' % j
1201 class P4RollBack(Command
):
1203 Command
.__init
__(self
)
1205 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1207 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1208 self
.rollbackLocalBranches
= False
1210 def run(self
, args
):
1213 maxChange
= int(args
[0])
1215 if "p4ExitCode" in p4Cmd("changes -m 1"):
1216 die("Problems executing p4");
1218 if self
.rollbackLocalBranches
:
1219 refPrefix
= "refs/heads/"
1220 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1222 refPrefix
= "refs/remotes/"
1223 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1226 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1228 ref
= refPrefix
+ line
1229 log
= extractLogMessageFromGitCommit(ref
)
1230 settings
= extractSettingsGitLog(log
)
1232 depotPaths
= settings
['depot-paths']
1233 change
= settings
['change']
1237 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1238 for p
in depotPaths
]))) == 0:
1239 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1240 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1243 while change
and int(change
) > maxChange
:
1246 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1247 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1248 log
= extractLogMessageFromGitCommit(ref
)
1249 settings
= extractSettingsGitLog(log
)
1252 depotPaths
= settings
['depot-paths']
1253 change
= settings
['change']
1256 print "%s rewound to %s" % (ref
, change
)
1260 class P4Submit(Command
, P4UserMap
):
1262 conflict_behavior_choices
= ("ask", "skip", "quit")
1265 Command
.__init
__(self
)
1266 P4UserMap
.__init
__(self
)
1268 optparse
.make_option("--origin", dest
="origin"),
1269 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1270 # preserve the user, requires relevant p4 permissions
1271 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1272 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1273 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1274 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1275 optparse
.make_option("--conflict", dest
="conflict_behavior",
1276 choices
=self
.conflict_behavior_choices
),
1277 optparse
.make_option("--branch", dest
="branch"),
1279 self
.description
= "Submit changes from git to the perforce depot."
1280 self
.usage
+= " [name of git branch to submit into perforce depot]"
1282 self
.detectRenames
= False
1283 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1284 self
.dry_run
= False
1285 self
.prepare_p4_only
= False
1286 self
.conflict_behavior
= None
1287 self
.isWindows
= (platform
.system() == "Windows")
1288 self
.exportLabels
= False
1289 self
.p4HasMoveCommand
= p4_has_move_command()
1292 if gitConfig('git-p4.largeFileSystem'):
1293 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1296 if len(p4CmdList("opened ...")) > 0:
1297 die("You have files opened with perforce! Close them before starting the sync.")
1299 def separate_jobs_from_description(self
, message
):
1300 """Extract and return a possible Jobs field in the commit
1301 message. It goes into a separate section in the p4 change
1304 A jobs line starts with "Jobs:" and looks like a new field
1305 in a form. Values are white-space separated on the same
1306 line or on following lines that start with a tab.
1308 This does not parse and extract the full git commit message
1309 like a p4 form. It just sees the Jobs: line as a marker
1310 to pass everything from then on directly into the p4 form,
1311 but outside the description section.
1313 Return a tuple (stripped log message, jobs string)."""
1315 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1317 return (message
, None)
1319 jobtext
= message
[m
.start():]
1320 stripped_message
= message
[:m
.start()].rstrip()
1321 return (stripped_message
, jobtext
)
1323 def prepareLogMessage(self
, template
, message
, jobs
):
1324 """Edits the template returned from "p4 change -o" to insert
1325 the message in the Description field, and the jobs text in
1329 inDescriptionSection
= False
1331 for line
in template
.split("\n"):
1332 if line
.startswith("#"):
1333 result
+= line
+ "\n"
1336 if inDescriptionSection
:
1337 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1338 inDescriptionSection
= False
1339 # insert Jobs section
1341 result
+= jobs
+ "\n"
1345 if line
.startswith("Description:"):
1346 inDescriptionSection
= True
1348 for messageLine
in message
.split("\n"):
1349 line
+= "\t" + messageLine
+ "\n"
1351 result
+= line
+ "\n"
1355 def patchRCSKeywords(self
, file, pattern
):
1356 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1357 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1359 outFile
= os
.fdopen(handle
, "w+")
1360 inFile
= open(file, "r")
1361 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1362 for line
in inFile
.readlines():
1363 line
= regexp
.sub(r
'$\1$', line
)
1367 # Forcibly overwrite the original file
1369 shutil
.move(outFileName
, file)
1371 # cleanup our temporary file
1372 os
.unlink(outFileName
)
1373 print "Failed to strip RCS keywords in %s" % file
1376 print "Patched up RCS keywords in %s" % file
1378 def p4UserForCommit(self
,id):
1379 # Return the tuple (perforce user,git email) for a given git commit id
1380 self
.getUserMapFromPerforceServer()
1381 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1382 "--format=%ae", id])
1383 gitEmail
= gitEmail
.strip()
1384 if not self
.emails
.has_key(gitEmail
):
1385 return (None,gitEmail
)
1387 return (self
.emails
[gitEmail
],gitEmail
)
1389 def checkValidP4Users(self
,commits
):
1390 # check if any git authors cannot be mapped to p4 users
1392 (user
,email
) = self
.p4UserForCommit(id)
1394 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1395 if gitConfigBool("git-p4.allowMissingP4Users"):
1398 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1400 def lastP4Changelist(self
):
1401 # Get back the last changelist number submitted in this client spec. This
1402 # then gets used to patch up the username in the change. If the same
1403 # client spec is being used by multiple processes then this might go
1405 results
= p4CmdList("client -o") # find the current client
1408 if r
.has_key('Client'):
1409 client
= r
['Client']
1412 die("could not get client spec")
1413 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1415 if r
.has_key('change'):
1417 die("Could not get changelist number for last submit - cannot patch up user details")
1419 def modifyChangelistUser(self
, changelist
, newUser
):
1420 # fixup the user field of a changelist after it has been submitted.
1421 changes
= p4CmdList("change -o %s" % changelist
)
1422 if len(changes
) != 1:
1423 die("Bad output from p4 change modifying %s to user %s" %
1424 (changelist
, newUser
))
1427 if c
['User'] == newUser
: return # nothing to do
1429 input = marshal
.dumps(c
)
1431 result
= p4CmdList("change -f -i", stdin
=input)
1433 if r
.has_key('code'):
1434 if r
['code'] == 'error':
1435 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1436 if r
.has_key('data'):
1437 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1439 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1441 def canChangeChangelists(self
):
1442 # check to see if we have p4 admin or super-user permissions, either of
1443 # which are required to modify changelists.
1444 results
= p4CmdList(["protects", self
.depotPath
])
1446 if r
.has_key('perm'):
1447 if r
['perm'] == 'admin':
1449 if r
['perm'] == 'super':
1453 def prepareSubmitTemplate(self
):
1454 """Run "p4 change -o" to grab a change specification template.
1455 This does not use "p4 -G", as it is nice to keep the submission
1456 template in original order, since a human might edit it.
1458 Remove lines in the Files section that show changes to files
1459 outside the depot path we're committing into."""
1461 [upstream
, settings
] = findUpstreamBranchPoint()
1464 inFilesSection
= False
1465 for line
in p4_read_pipe_lines(['change', '-o']):
1466 if line
.endswith("\r\n"):
1467 line
= line
[:-2] + "\n"
1469 if line
.startswith("\t"):
1470 # path starts and ends with a tab
1472 lastTab
= path
.rfind("\t")
1474 path
= path
[:lastTab
]
1475 if settings
.has_key('depot-paths'):
1476 if not [p
for p
in settings
['depot-paths']
1477 if p4PathStartsWith(path
, p
)]:
1480 if not p4PathStartsWith(path
, self
.depotPath
):
1483 inFilesSection
= False
1485 if line
.startswith("Files:"):
1486 inFilesSection
= True
1492 def edit_template(self
, template_file
):
1493 """Invoke the editor to let the user change the submission
1494 message. Return true if okay to continue with the submit."""
1496 # if configured to skip the editing part, just submit
1497 if gitConfigBool("git-p4.skipSubmitEdit"):
1500 # look at the modification time, to check later if the user saved
1502 mtime
= os
.stat(template_file
).st_mtime
1505 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1506 editor
= os
.environ
.get("P4EDITOR")
1508 editor
= read_pipe("git var GIT_EDITOR").strip()
1509 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1511 # If the file was not saved, prompt to see if this patch should
1512 # be skipped. But skip this verification step if configured so.
1513 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1516 # modification time updated means user saved the file
1517 if os
.stat(template_file
).st_mtime
> mtime
:
1521 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1527 def get_diff_description(self
, editedFiles
, filesToAdd
):
1529 if os
.environ
.has_key("P4DIFF"):
1530 del(os
.environ
["P4DIFF"])
1532 for editedFile
in editedFiles
:
1533 diff
+= p4_read_pipe(['diff', '-du',
1534 wildcard_encode(editedFile
)])
1538 for newFile
in filesToAdd
:
1539 newdiff
+= "==== new file ====\n"
1540 newdiff
+= "--- /dev/null\n"
1541 newdiff
+= "+++ %s\n" % newFile
1542 f
= open(newFile
, "r")
1543 for line
in f
.readlines():
1544 newdiff
+= "+" + line
1547 return (diff
+ newdiff
).replace('\r\n', '\n')
1549 def applyCommit(self
, id):
1550 """Apply one commit, return True if it succeeded."""
1552 print "Applying", read_pipe(["git", "show", "-s",
1553 "--format=format:%h %s", id])
1555 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1557 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1559 filesToDelete
= set()
1561 pureRenameCopy
= set()
1562 filesToChangeExecBit
= {}
1565 diff
= parseDiffTreeEntry(line
)
1566 modifier
= diff
['status']
1570 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1571 filesToChangeExecBit
[path
] = diff
['dst_mode']
1572 editedFiles
.add(path
)
1573 elif modifier
== "A":
1574 filesToAdd
.add(path
)
1575 filesToChangeExecBit
[path
] = diff
['dst_mode']
1576 if path
in filesToDelete
:
1577 filesToDelete
.remove(path
)
1578 elif modifier
== "D":
1579 filesToDelete
.add(path
)
1580 if path
in filesToAdd
:
1581 filesToAdd
.remove(path
)
1582 elif modifier
== "C":
1583 src
, dest
= diff
['src'], diff
['dst']
1584 p4_integrate(src
, dest
)
1585 pureRenameCopy
.add(dest
)
1586 if diff
['src_sha1'] != diff
['dst_sha1']:
1588 pureRenameCopy
.discard(dest
)
1589 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1591 pureRenameCopy
.discard(dest
)
1592 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1594 # turn off read-only attribute
1595 os
.chmod(dest
, stat
.S_IWRITE
)
1597 editedFiles
.add(dest
)
1598 elif modifier
== "R":
1599 src
, dest
= diff
['src'], diff
['dst']
1600 if self
.p4HasMoveCommand
:
1601 p4_edit(src
) # src must be open before move
1602 p4_move(src
, dest
) # opens for (move/delete, move/add)
1604 p4_integrate(src
, dest
)
1605 if diff
['src_sha1'] != diff
['dst_sha1']:
1608 pureRenameCopy
.add(dest
)
1609 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1610 if not self
.p4HasMoveCommand
:
1611 p4_edit(dest
) # with move: already open, writable
1612 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1613 if not self
.p4HasMoveCommand
:
1615 os
.chmod(dest
, stat
.S_IWRITE
)
1617 filesToDelete
.add(src
)
1618 editedFiles
.add(dest
)
1620 die("unknown modifier %s for %s" % (modifier
, path
))
1622 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1623 patchcmd
= diffcmd
+ " | git apply "
1624 tryPatchCmd
= patchcmd
+ "--check -"
1625 applyPatchCmd
= patchcmd
+ "--check --apply -"
1626 patch_succeeded
= True
1628 if os
.system(tryPatchCmd
) != 0:
1629 fixed_rcs_keywords
= False
1630 patch_succeeded
= False
1631 print "Unfortunately applying the change failed!"
1633 # Patch failed, maybe it's just RCS keyword woes. Look through
1634 # the patch to see if that's possible.
1635 if gitConfigBool("git-p4.attemptRCSCleanup"):
1639 for file in editedFiles | filesToDelete
:
1640 # did this file's delta contain RCS keywords?
1641 pattern
= p4_keywords_regexp_for_file(file)
1644 # this file is a possibility...look for RCS keywords.
1645 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1646 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1647 if regexp
.search(line
):
1649 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1650 kwfiles
[file] = pattern
1653 for file in kwfiles
:
1655 print "zapping %s with %s" % (line
,pattern
)
1656 # File is being deleted, so not open in p4. Must
1657 # disable the read-only bit on windows.
1658 if self
.isWindows
and file not in editedFiles
:
1659 os
.chmod(file, stat
.S_IWRITE
)
1660 self
.patchRCSKeywords(file, kwfiles
[file])
1661 fixed_rcs_keywords
= True
1663 if fixed_rcs_keywords
:
1664 print "Retrying the patch with RCS keywords cleaned up"
1665 if os
.system(tryPatchCmd
) == 0:
1666 patch_succeeded
= True
1668 if not patch_succeeded
:
1669 for f
in editedFiles
:
1674 # Apply the patch for real, and do add/delete/+x handling.
1676 system(applyPatchCmd
)
1678 for f
in filesToAdd
:
1680 for f
in filesToDelete
:
1684 # Set/clear executable bits
1685 for f
in filesToChangeExecBit
.keys():
1686 mode
= filesToChangeExecBit
[f
]
1687 setP4ExecBit(f
, mode
)
1690 # Build p4 change description, starting with the contents
1691 # of the git commit message.
1693 logMessage
= extractLogMessageFromGitCommit(id)
1694 logMessage
= logMessage
.strip()
1695 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1697 template
= self
.prepareSubmitTemplate()
1698 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1700 if self
.preserveUser
:
1701 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1703 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1704 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1705 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1706 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1708 separatorLine
= "######## everything below this line is just the diff #######\n"
1709 if not self
.prepare_p4_only
:
1710 submitTemplate
+= separatorLine
1711 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
)
1713 (handle
, fileName
) = tempfile
.mkstemp()
1714 tmpFile
= os
.fdopen(handle
, "w+b")
1716 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1717 tmpFile
.write(submitTemplate
)
1720 if self
.prepare_p4_only
:
1722 # Leave the p4 tree prepared, and the submit template around
1723 # and let the user decide what to do next
1726 print "P4 workspace prepared for submission."
1727 print "To submit or revert, go to client workspace"
1728 print " " + self
.clientPath
1730 print "To submit, use \"p4 submit\" to write a new description,"
1731 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1732 " \"git p4\"." % fileName
1733 print "You can delete the file \"%s\" when finished." % fileName
1735 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1736 print "To preserve change ownership by user %s, you must\n" \
1737 "do \"p4 change -f <change>\" after submitting and\n" \
1738 "edit the User field."
1740 print "After submitting, renamed files must be re-synced."
1741 print "Invoke \"p4 sync -f\" on each of these files:"
1742 for f
in pureRenameCopy
:
1746 print "To revert the changes, use \"p4 revert ...\", and delete"
1747 print "the submit template file \"%s\"" % fileName
1749 print "Since the commit adds new files, they must be deleted:"
1750 for f
in filesToAdd
:
1756 # Let the user edit the change description, then submit it.
1761 if self
.edit_template(fileName
):
1762 # read the edited message and submit
1763 tmpFile
= open(fileName
, "rb")
1764 message
= tmpFile
.read()
1767 message
= message
.replace("\r\n", "\n")
1768 submitTemplate
= message
[:message
.index(separatorLine
)]
1769 p4_write_pipe(['submit', '-i'], submitTemplate
)
1771 if self
.preserveUser
:
1773 # Get last changelist number. Cannot easily get it from
1774 # the submit command output as the output is
1776 changelist
= self
.lastP4Changelist()
1777 self
.modifyChangelistUser(changelist
, p4User
)
1779 # The rename/copy happened by applying a patch that created a
1780 # new file. This leaves it writable, which confuses p4.
1781 for f
in pureRenameCopy
:
1788 print "Submission cancelled, undoing p4 changes."
1789 for f
in editedFiles
:
1791 for f
in filesToAdd
:
1794 for f
in filesToDelete
:
1800 # Export git tags as p4 labels. Create a p4 label and then tag
1802 def exportGitTags(self
, gitTags
):
1803 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1804 if len(validLabelRegexp
) == 0:
1805 validLabelRegexp
= defaultLabelRegexp
1806 m
= re
.compile(validLabelRegexp
)
1808 for name
in gitTags
:
1810 if not m
.match(name
):
1812 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1815 # Get the p4 commit this corresponds to
1816 logMessage
= extractLogMessageFromGitCommit(name
)
1817 values
= extractSettingsGitLog(logMessage
)
1819 if not values
.has_key('change'):
1820 # a tag pointing to something not sent to p4; ignore
1822 print "git tag %s does not give a p4 commit" % name
1825 changelist
= values
['change']
1827 # Get the tag details.
1831 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1834 if re
.match(r
'tag\s+', l
):
1836 elif re
.match(r
'\s*$', l
):
1843 body
= ["lightweight tag imported by git p4\n"]
1845 # Create the label - use the same view as the client spec we are using
1846 clientSpec
= getClientSpec()
1848 labelTemplate
= "Label: %s\n" % name
1849 labelTemplate
+= "Description:\n"
1851 labelTemplate
+= "\t" + b
+ "\n"
1852 labelTemplate
+= "View:\n"
1853 for depot_side
in clientSpec
.mappings
:
1854 labelTemplate
+= "\t%s\n" % depot_side
1857 print "Would create p4 label %s for tag" % name
1858 elif self
.prepare_p4_only
:
1859 print "Not creating p4 label %s for tag due to option" \
1860 " --prepare-p4-only" % name
1862 p4_write_pipe(["label", "-i"], labelTemplate
)
1865 p4_system(["tag", "-l", name
] +
1866 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
1869 print "created p4 label for tag %s" % name
1871 def run(self
, args
):
1873 self
.master
= currentGitBranch()
1874 elif len(args
) == 1:
1875 self
.master
= args
[0]
1876 if not branchExists(self
.master
):
1877 die("Branch %s does not exist" % self
.master
)
1882 allowSubmit
= gitConfig("git-p4.allowSubmit")
1883 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1884 die("%s is not in git-p4.allowSubmit" % self
.master
)
1886 [upstream
, settings
] = findUpstreamBranchPoint()
1887 self
.depotPath
= settings
['depot-paths'][0]
1888 if len(self
.origin
) == 0:
1889 self
.origin
= upstream
1891 if self
.preserveUser
:
1892 if not self
.canChangeChangelists():
1893 die("Cannot preserve user names without p4 super-user or admin permissions")
1895 # if not set from the command line, try the config file
1896 if self
.conflict_behavior
is None:
1897 val
= gitConfig("git-p4.conflict")
1899 if val
not in self
.conflict_behavior_choices
:
1900 die("Invalid value '%s' for config git-p4.conflict" % val
)
1903 self
.conflict_behavior
= val
1906 print "Origin branch is " + self
.origin
1908 if len(self
.depotPath
) == 0:
1909 print "Internal error: cannot locate perforce depot path from existing branches"
1912 self
.useClientSpec
= False
1913 if gitConfigBool("git-p4.useclientspec"):
1914 self
.useClientSpec
= True
1915 if self
.useClientSpec
:
1916 self
.clientSpecDirs
= getClientSpec()
1918 # Check for the existance of P4 branches
1919 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
1921 if self
.useClientSpec
and not branchesDetected
:
1922 # all files are relative to the client spec
1923 self
.clientPath
= getClientRoot()
1925 self
.clientPath
= p4Where(self
.depotPath
)
1927 if self
.clientPath
== "":
1928 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1930 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1931 self
.oldWorkingDirectory
= os
.getcwd()
1933 # ensure the clientPath exists
1934 new_client_dir
= False
1935 if not os
.path
.exists(self
.clientPath
):
1936 new_client_dir
= True
1937 os
.makedirs(self
.clientPath
)
1939 chdir(self
.clientPath
, is_client_path
=True)
1941 print "Would synchronize p4 checkout in %s" % self
.clientPath
1943 print "Synchronizing p4 checkout..."
1945 # old one was destroyed, and maybe nobody told p4
1946 p4_sync("...", "-f")
1953 commitish
= self
.master
1957 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, commitish
)]):
1958 commits
.append(line
.strip())
1961 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
1962 self
.checkAuthorship
= False
1964 self
.checkAuthorship
= True
1966 if self
.preserveUser
:
1967 self
.checkValidP4Users(commits
)
1970 # Build up a set of options to be passed to diff when
1971 # submitting each commit to p4.
1973 if self
.detectRenames
:
1974 # command-line -M arg
1975 self
.diffOpts
= "-M"
1977 # If not explicitly set check the config variable
1978 detectRenames
= gitConfig("git-p4.detectRenames")
1980 if detectRenames
.lower() == "false" or detectRenames
== "":
1982 elif detectRenames
.lower() == "true":
1983 self
.diffOpts
= "-M"
1985 self
.diffOpts
= "-M%s" % detectRenames
1987 # no command-line arg for -C or --find-copies-harder, just
1989 detectCopies
= gitConfig("git-p4.detectCopies")
1990 if detectCopies
.lower() == "false" or detectCopies
== "":
1992 elif detectCopies
.lower() == "true":
1993 self
.diffOpts
+= " -C"
1995 self
.diffOpts
+= " -C%s" % detectCopies
1997 if gitConfigBool("git-p4.detectCopiesHarder"):
1998 self
.diffOpts
+= " --find-copies-harder"
2001 # Apply the commits, one at a time. On failure, ask if should
2002 # continue to try the rest of the patches, or quit.
2007 last
= len(commits
) - 1
2008 for i
, commit
in enumerate(commits
):
2010 print " ", read_pipe(["git", "show", "-s",
2011 "--format=format:%h %s", commit
])
2014 ok
= self
.applyCommit(commit
)
2016 applied
.append(commit
)
2018 if self
.prepare_p4_only
and i
< last
:
2019 print "Processing only the first commit due to option" \
2020 " --prepare-p4-only"
2025 # prompt for what to do, or use the option/variable
2026 if self
.conflict_behavior
== "ask":
2027 print "What do you want to do?"
2028 response
= raw_input("[s]kip this commit but apply"
2029 " the rest, or [q]uit? ")
2032 elif self
.conflict_behavior
== "skip":
2034 elif self
.conflict_behavior
== "quit":
2037 die("Unknown conflict_behavior '%s'" %
2038 self
.conflict_behavior
)
2040 if response
[0] == "s":
2041 print "Skipping this commit, but applying the rest"
2043 if response
[0] == "q":
2050 chdir(self
.oldWorkingDirectory
)
2054 elif self
.prepare_p4_only
:
2056 elif len(commits
) == len(applied
):
2057 print "All commits applied!"
2061 sync
.branch
= self
.branch
2068 if len(applied
) == 0:
2069 print "No commits applied."
2071 print "Applied only the commits marked with '*':"
2077 print star
, read_pipe(["git", "show", "-s",
2078 "--format=format:%h %s", c
])
2079 print "You will have to do 'git p4 sync' and rebase."
2081 if gitConfigBool("git-p4.exportLabels"):
2082 self
.exportLabels
= True
2084 if self
.exportLabels
:
2085 p4Labels
= getP4Labels(self
.depotPath
)
2086 gitTags
= getGitTags()
2088 missingGitTags
= gitTags
- p4Labels
2089 self
.exportGitTags(missingGitTags
)
2091 # exit with error unless everything applied perfectly
2092 if len(commits
) != len(applied
):
2098 """Represent a p4 view ("p4 help views"), and map files in a
2099 repo according to the view."""
2101 def __init__(self
, client_name
):
2103 self
.client_prefix
= "//%s/" % client_name
2104 # cache results of "p4 where" to lookup client file locations
2105 self
.client_spec_path_cache
= {}
2107 def append(self
, view_line
):
2108 """Parse a view line, splitting it into depot and client
2109 sides. Append to self.mappings, preserving order. This
2110 is only needed for tag creation."""
2112 # Split the view line into exactly two words. P4 enforces
2113 # structure on these lines that simplifies this quite a bit.
2115 # Either or both words may be double-quoted.
2116 # Single quotes do not matter.
2117 # Double-quote marks cannot occur inside the words.
2118 # A + or - prefix is also inside the quotes.
2119 # There are no quotes unless they contain a space.
2120 # The line is already white-space stripped.
2121 # The two words are separated by a single space.
2123 if view_line
[0] == '"':
2124 # First word is double quoted. Find its end.
2125 close_quote_index
= view_line
.find('"', 1)
2126 if close_quote_index
<= 0:
2127 die("No first-word closing quote found: %s" % view_line
)
2128 depot_side
= view_line
[1:close_quote_index
]
2129 # skip closing quote and space
2130 rhs_index
= close_quote_index
+ 1 + 1
2132 space_index
= view_line
.find(" ")
2133 if space_index
<= 0:
2134 die("No word-splitting space found: %s" % view_line
)
2135 depot_side
= view_line
[0:space_index
]
2136 rhs_index
= space_index
+ 1
2138 # prefix + means overlay on previous mapping
2139 if depot_side
.startswith("+"):
2140 depot_side
= depot_side
[1:]
2142 # prefix - means exclude this path, leave out of mappings
2144 if depot_side
.startswith("-"):
2146 depot_side
= depot_side
[1:]
2149 self
.mappings
.append(depot_side
)
2151 def convert_client_path(self
, clientFile
):
2152 # chop off //client/ part to make it relative
2153 if not clientFile
.startswith(self
.client_prefix
):
2154 die("No prefix '%s' on clientFile '%s'" %
2155 (self
.client_prefix
, clientFile
))
2156 return clientFile
[len(self
.client_prefix
):]
2158 def update_client_spec_path_cache(self
, files
):
2159 """ Caching file paths by "p4 where" batch query """
2161 # List depot file paths exclude that already cached
2162 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2164 if len(fileArgs
) == 0:
2165 return # All files in cache
2167 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2168 for res
in where_result
:
2169 if "code" in res
and res
["code"] == "error":
2170 # assume error is "... file(s) not in client view"
2172 if "clientFile" not in res
:
2173 die("No clientFile in 'p4 where' output")
2175 # it will list all of them, but only one not unmap-ped
2177 if gitConfigBool("core.ignorecase"):
2178 res
['depotFile'] = res
['depotFile'].lower()
2179 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2181 # not found files or unmap files set to ""
2182 for depotFile
in fileArgs
:
2183 if gitConfigBool("core.ignorecase"):
2184 depotFile
= depotFile
.lower()
2185 if depotFile
not in self
.client_spec_path_cache
:
2186 self
.client_spec_path_cache
[depotFile
] = ""
2188 def map_in_client(self
, depot_path
):
2189 """Return the relative location in the client where this
2190 depot file should live. Returns "" if the file should
2191 not be mapped in the client."""
2193 if gitConfigBool("core.ignorecase"):
2194 depot_path
= depot_path
.lower()
2196 if depot_path
in self
.client_spec_path_cache
:
2197 return self
.client_spec_path_cache
[depot_path
]
2199 die( "Error: %s is not found in client spec path" % depot_path
)
2202 class P4Sync(Command
, P4UserMap
):
2203 delete_actions
= ( "delete", "move/delete", "purge" )
2206 Command
.__init
__(self
)
2207 P4UserMap
.__init
__(self
)
2209 optparse
.make_option("--branch", dest
="branch"),
2210 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2211 optparse
.make_option("--changesfile", dest
="changesFile"),
2212 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2213 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2214 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2215 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2216 help="Import into refs/heads/ , not refs/remotes"),
2217 optparse
.make_option("--max-changes", dest
="maxChanges",
2218 help="Maximum number of changes to import"),
2219 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2220 help="Internal block size to use when iteratively calling p4 changes"),
2221 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2222 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2223 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2224 help="Only sync files that are included in the Perforce Client Spec"),
2225 optparse
.make_option("-/", dest
="cloneExclude",
2226 action
="append", type="string",
2227 help="exclude depot path"),
2229 self
.description
= """Imports from Perforce into a git repository.\n
2231 //depot/my/project/ -- to import the current head
2232 //depot/my/project/@all -- to import everything
2233 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2235 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2237 self
.usage
+= " //depot/path[@revRange]"
2239 self
.createdBranches
= set()
2240 self
.committedChanges
= set()
2242 self
.detectBranches
= False
2243 self
.detectLabels
= False
2244 self
.importLabels
= False
2245 self
.changesFile
= ""
2246 self
.syncWithOrigin
= True
2247 self
.importIntoRemotes
= True
2248 self
.maxChanges
= ""
2249 self
.changes_block_size
= None
2250 self
.keepRepoPath
= False
2251 self
.depotPaths
= None
2252 self
.p4BranchesInGit
= []
2253 self
.cloneExclude
= []
2254 self
.useClientSpec
= False
2255 self
.useClientSpec_from_options
= False
2256 self
.clientSpecDirs
= None
2257 self
.tempBranches
= []
2258 self
.tempBranchLocation
= "git-p4-tmp"
2259 self
.largeFileSystem
= None
2261 if gitConfig('git-p4.largeFileSystem'):
2262 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2263 self
.largeFileSystem
= largeFileSystemConstructor(
2264 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2267 if gitConfig("git-p4.syncFromOrigin") == "false":
2268 self
.syncWithOrigin
= False
2270 # This is required for the "append" cloneExclude action
2271 def ensure_value(self
, attr
, value
):
2272 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2273 setattr(self
, attr
, value
)
2274 return getattr(self
, attr
)
2276 # Force a checkpoint in fast-import and wait for it to finish
2277 def checkpoint(self
):
2278 self
.gitStream
.write("checkpoint\n\n")
2279 self
.gitStream
.write("progress checkpoint\n\n")
2280 out
= self
.gitOutput
.readline()
2282 print "checkpoint finished: " + out
2284 def extractFilesFromCommit(self
, commit
):
2285 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2286 for path
in self
.cloneExclude
]
2289 while commit
.has_key("depotFile%s" % fnum
):
2290 path
= commit
["depotFile%s" % fnum
]
2292 if [p
for p
in self
.cloneExclude
2293 if p4PathStartsWith(path
, p
)]:
2296 found
= [p
for p
in self
.depotPaths
2297 if p4PathStartsWith(path
, p
)]
2304 file["rev"] = commit
["rev%s" % fnum
]
2305 file["action"] = commit
["action%s" % fnum
]
2306 file["type"] = commit
["type%s" % fnum
]
2311 def stripRepoPath(self
, path
, prefixes
):
2312 """When streaming files, this is called to map a p4 depot path
2313 to where it should go in git. The prefixes are either
2314 self.depotPaths, or self.branchPrefixes in the case of
2315 branch detection."""
2317 if self
.useClientSpec
:
2318 # branch detection moves files up a level (the branch name)
2319 # from what client spec interpretation gives
2320 path
= self
.clientSpecDirs
.map_in_client(path
)
2321 if self
.detectBranches
:
2322 for b
in self
.knownBranches
:
2323 if path
.startswith(b
+ "/"):
2324 path
= path
[len(b
)+1:]
2326 elif self
.keepRepoPath
:
2327 # Preserve everything in relative path name except leading
2328 # //depot/; just look at first prefix as they all should
2329 # be in the same depot.
2330 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2331 if p4PathStartsWith(path
, depot
):
2332 path
= path
[len(depot
):]
2336 if p4PathStartsWith(path
, p
):
2337 path
= path
[len(p
):]
2340 path
= wildcard_decode(path
)
2343 def splitFilesIntoBranches(self
, commit
):
2344 """Look at each depotFile in the commit to figure out to what
2345 branch it belongs."""
2347 if self
.clientSpecDirs
:
2348 files
= self
.extractFilesFromCommit(commit
)
2349 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2353 while commit
.has_key("depotFile%s" % fnum
):
2354 path
= commit
["depotFile%s" % fnum
]
2355 found
= [p
for p
in self
.depotPaths
2356 if p4PathStartsWith(path
, p
)]
2363 file["rev"] = commit
["rev%s" % fnum
]
2364 file["action"] = commit
["action%s" % fnum
]
2365 file["type"] = commit
["type%s" % fnum
]
2368 # start with the full relative path where this file would
2370 if self
.useClientSpec
:
2371 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2373 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2375 for branch
in self
.knownBranches
.keys():
2376 # add a trailing slash so that a commit into qt/4.2foo
2377 # doesn't end up in qt/4.2, e.g.
2378 if relPath
.startswith(branch
+ "/"):
2379 if branch
not in branches
:
2380 branches
[branch
] = []
2381 branches
[branch
].append(file)
2386 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2387 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2388 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2390 self
.gitStream
.write(d
)
2391 self
.gitStream
.write('\n')
2393 # output one file from the P4 stream
2394 # - helper for streamP4Files
2396 def streamOneP4File(self
, file, contents
):
2397 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2399 size
= int(self
.stream_file
['fileSize'])
2400 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2403 (type_base
, type_mods
) = split_p4_type(file["type"])
2406 if "x" in type_mods
:
2408 if type_base
== "symlink":
2410 # p4 print on a symlink sometimes contains "target\n";
2411 # if it does, remove the newline
2412 data
= ''.join(contents
)
2414 # Some version of p4 allowed creating a symlink that pointed
2415 # to nothing. This causes p4 errors when checking out such
2416 # a change, and errors here too. Work around it by ignoring
2417 # the bad symlink; hopefully a future change fixes it.
2418 print "\nIgnoring empty symlink in %s" % file['depotFile']
2420 elif data
[-1] == '\n':
2421 contents
= [data
[:-1]]
2425 if type_base
== "utf16":
2426 # p4 delivers different text in the python output to -G
2427 # than it does when using "print -o", or normal p4 client
2428 # operations. utf16 is converted to ascii or utf8, perhaps.
2429 # But ascii text saved as -t utf16 is completely mangled.
2430 # Invoke print -o to get the real contents.
2432 # On windows, the newlines will always be mangled by print, so put
2433 # them back too. This is not needed to the cygwin windows version,
2434 # just the native "NT" type.
2437 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2438 except Exception as e
:
2439 if 'Translation of file content failed' in str(e
):
2440 type_base
= 'binary'
2444 if p4_version_string().find('/NT') >= 0:
2445 text
= text
.replace('\r\n', '\n')
2448 if type_base
== "apple":
2449 # Apple filetype files will be streamed as a concatenation of
2450 # its appledouble header and the contents. This is useless
2451 # on both macs and non-macs. If using "print -q -o xx", it
2452 # will create "xx" with the data, and "%xx" with the header.
2453 # This is also not very useful.
2455 # Ideally, someday, this script can learn how to generate
2456 # appledouble files directly and import those to git, but
2457 # non-mac machines can never find a use for apple filetype.
2458 print "\nIgnoring apple filetype file %s" % file['depotFile']
2461 # Note that we do not try to de-mangle keywords on utf16 files,
2462 # even though in theory somebody may want that.
2463 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2465 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2466 text
= ''.join(contents
)
2467 text
= regexp
.sub(r
'$\1$', text
)
2471 relPath
.decode('ascii')
2474 if gitConfig('git-p4.pathEncoding'):
2475 encoding
= gitConfig('git-p4.pathEncoding')
2476 relPath
= relPath
.decode(encoding
, 'replace').encode('utf8', 'replace')
2478 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, relPath
)
2480 if self
.largeFileSystem
:
2481 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2483 self
.writeToGitStream(git_mode
, relPath
, contents
)
2485 def streamOneP4Deletion(self
, file):
2486 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2488 sys
.stdout
.write("delete %s\n" % relPath
)
2490 self
.gitStream
.write("D %s\n" % relPath
)
2492 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2493 self
.largeFileSystem
.removeLargeFile(relPath
)
2495 # handle another chunk of streaming data
2496 def streamP4FilesCb(self
, marshalled
):
2498 # catch p4 errors and complain
2500 if "code" in marshalled
:
2501 if marshalled
["code"] == "error":
2502 if "data" in marshalled
:
2503 err
= marshalled
["data"].rstrip()
2505 if not err
and 'fileSize' in self
.stream_file
:
2506 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2507 if required_bytes
> 0:
2508 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2509 os
.getcwd(), required_bytes
/1024/1024
2514 if self
.stream_have_file_info
:
2515 if "depotFile" in self
.stream_file
:
2516 f
= self
.stream_file
["depotFile"]
2517 # force a failure in fast-import, else an empty
2518 # commit will be made
2519 self
.gitStream
.write("\n")
2520 self
.gitStream
.write("die-now\n")
2521 self
.gitStream
.close()
2522 # ignore errors, but make sure it exits first
2523 self
.importProcess
.wait()
2525 die("Error from p4 print for %s: %s" % (f
, err
))
2527 die("Error from p4 print: %s" % err
)
2529 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2530 # start of a new file - output the old one first
2531 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2532 self
.stream_file
= {}
2533 self
.stream_contents
= []
2534 self
.stream_have_file_info
= False
2536 # pick up the new file information... for the
2537 # 'data' field we need to append to our array
2538 for k
in marshalled
.keys():
2540 if 'streamContentSize' not in self
.stream_file
:
2541 self
.stream_file
['streamContentSize'] = 0
2542 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2543 self
.stream_contents
.append(marshalled
['data'])
2545 self
.stream_file
[k
] = marshalled
[k
]
2548 'streamContentSize' in self
.stream_file
and
2549 'fileSize' in self
.stream_file
and
2550 'depotFile' in self
.stream_file
):
2551 size
= int(self
.stream_file
["fileSize"])
2553 progress
= 100*self
.stream_file
['streamContentSize']/size
2554 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2557 self
.stream_have_file_info
= True
2559 # Stream directly from "p4 files" into "git fast-import"
2560 def streamP4Files(self
, files
):
2566 # if using a client spec, only add the files that have
2567 # a path in the client
2568 if self
.clientSpecDirs
:
2569 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
2572 filesForCommit
.append(f
)
2573 if f
['action'] in self
.delete_actions
:
2574 filesToDelete
.append(f
)
2576 filesToRead
.append(f
)
2579 for f
in filesToDelete
:
2580 self
.streamOneP4Deletion(f
)
2582 if len(filesToRead
) > 0:
2583 self
.stream_file
= {}
2584 self
.stream_contents
= []
2585 self
.stream_have_file_info
= False
2587 # curry self argument
2588 def streamP4FilesCbSelf(entry
):
2589 self
.streamP4FilesCb(entry
)
2591 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2593 p4CmdList(["-x", "-", "print"],
2595 cb
=streamP4FilesCbSelf
)
2598 if self
.stream_file
.has_key('depotFile'):
2599 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2601 def make_email(self
, userid
):
2602 if userid
in self
.users
:
2603 return self
.users
[userid
]
2605 return "%s <a@b>" % userid
2607 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2608 """ Stream a p4 tag.
2609 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2613 print "writing tag %s for commit %s" % (labelName
, commit
)
2614 gitStream
.write("tag %s\n" % labelName
)
2615 gitStream
.write("from %s\n" % commit
)
2617 if labelDetails
.has_key('Owner'):
2618 owner
= labelDetails
["Owner"]
2622 # Try to use the owner of the p4 label, or failing that,
2623 # the current p4 user id.
2625 email
= self
.make_email(owner
)
2627 email
= self
.make_email(self
.p4UserId())
2628 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2630 gitStream
.write("tagger %s\n" % tagger
)
2632 print "labelDetails=",labelDetails
2633 if labelDetails
.has_key('Description'):
2634 description
= labelDetails
['Description']
2636 description
= 'Label from git p4'
2638 gitStream
.write("data %d\n" % len(description
))
2639 gitStream
.write(description
)
2640 gitStream
.write("\n")
2642 def commit(self
, details
, files
, branch
, parent
= ""):
2643 epoch
= details
["time"]
2644 author
= details
["user"]
2647 print "commit into %s" % branch
2649 # start with reading files; if that fails, we should not
2653 if [p
for p
in self
.branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2654 new_files
.append (f
)
2656 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2658 if self
.clientSpecDirs
:
2659 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2661 self
.gitStream
.write("commit %s\n" % branch
)
2662 self
.gitStream
.write("mark :%s\n" % details
["change"])
2663 self
.committedChanges
.add(int(details
["change"]))
2665 if author
not in self
.users
:
2666 self
.getUserMapFromPerforceServer()
2667 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2669 self
.gitStream
.write("committer %s\n" % committer
)
2671 self
.gitStream
.write("data <<EOT\n")
2672 self
.gitStream
.write(details
["desc"])
2673 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2674 (','.join(self
.branchPrefixes
), details
["change"]))
2675 if len(details
['options']) > 0:
2676 self
.gitStream
.write(": options = %s" % details
['options'])
2677 self
.gitStream
.write("]\nEOT\n\n")
2681 print "parent %s" % parent
2682 self
.gitStream
.write("from %s\n" % parent
)
2684 self
.streamP4Files(new_files
)
2685 self
.gitStream
.write("\n")
2687 change
= int(details
["change"])
2689 if self
.labels
.has_key(change
):
2690 label
= self
.labels
[change
]
2691 labelDetails
= label
[0]
2692 labelRevisions
= label
[1]
2694 print "Change %s is labelled %s" % (change
, labelDetails
)
2696 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2697 for p
in self
.branchPrefixes
])
2699 if len(files
) == len(labelRevisions
):
2703 if info
["action"] in self
.delete_actions
:
2705 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2707 if cleanedFiles
== labelRevisions
:
2708 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2712 print ("Tag %s does not match with change %s: files do not match."
2713 % (labelDetails
["label"], change
))
2717 print ("Tag %s does not match with change %s: file count is different."
2718 % (labelDetails
["label"], change
))
2720 # Build a dictionary of changelists and labels, for "detect-labels" option.
2721 def getLabels(self
):
2724 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2725 if len(l
) > 0 and not self
.silent
:
2726 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2729 label
= output
["label"]
2733 print "Querying files for label %s" % label
2734 for file in p4CmdList(["files"] +
2735 ["%s...@%s" % (p
, label
)
2736 for p
in self
.depotPaths
]):
2737 revisions
[file["depotFile"]] = file["rev"]
2738 change
= int(file["change"])
2739 if change
> newestChange
:
2740 newestChange
= change
2742 self
.labels
[newestChange
] = [output
, revisions
]
2745 print "Label changes: %s" % self
.labels
.keys()
2747 # Import p4 labels as git tags. A direct mapping does not
2748 # exist, so assume that if all the files are at the same revision
2749 # then we can use that, or it's something more complicated we should
2751 def importP4Labels(self
, stream
, p4Labels
):
2753 print "import p4 labels: " + ' '.join(p4Labels
)
2755 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2756 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2757 if len(validLabelRegexp
) == 0:
2758 validLabelRegexp
= defaultLabelRegexp
2759 m
= re
.compile(validLabelRegexp
)
2761 for name
in p4Labels
:
2764 if not m
.match(name
):
2766 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2769 if name
in ignoredP4Labels
:
2772 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2774 # get the most recent changelist for each file in this label
2775 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2776 for p
in self
.depotPaths
])
2778 if change
.has_key('change'):
2779 # find the corresponding git commit; take the oldest commit
2780 changelist
= int(change
['change'])
2781 if changelist
in self
.committedChanges
:
2782 gitCommit
= ":%d" % changelist
# use a fast-import mark
2785 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2786 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
2787 if len(gitCommit
) == 0:
2788 print "importing label %s: could not find git commit for changelist %d" % (name
, changelist
)
2791 gitCommit
= gitCommit
.strip()
2794 # Convert from p4 time format
2796 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2798 print "Could not convert label time %s" % labelDetails
['Update']
2801 when
= int(time
.mktime(tmwhen
))
2802 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2804 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2807 print "Label %s has no changelists - possibly deleted?" % name
2810 # We can't import this label; don't try again as it will get very
2811 # expensive repeatedly fetching all the files for labels that will
2812 # never be imported. If the label is moved in the future, the
2813 # ignore will need to be removed manually.
2814 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2816 def guessProjectName(self
):
2817 for p
in self
.depotPaths
:
2820 p
= p
[p
.strip().rfind("/") + 1:]
2821 if not p
.endswith("/"):
2825 def getBranchMapping(self
):
2826 lostAndFoundBranches
= set()
2828 user
= gitConfig("git-p4.branchUser")
2830 command
= "branches -u %s" % user
2832 command
= "branches"
2834 for info
in p4CmdList(command
):
2835 details
= p4Cmd(["branch", "-o", info
["branch"]])
2837 while details
.has_key("View%s" % viewIdx
):
2838 paths
= details
["View%s" % viewIdx
].split(" ")
2839 viewIdx
= viewIdx
+ 1
2840 # require standard //depot/foo/... //depot/bar/... mapping
2841 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2844 destination
= paths
[1]
2846 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2847 source
= source
[len(self
.depotPaths
[0]):-4]
2848 destination
= destination
[len(self
.depotPaths
[0]):-4]
2850 if destination
in self
.knownBranches
:
2852 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2853 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2856 self
.knownBranches
[destination
] = source
2858 lostAndFoundBranches
.discard(destination
)
2860 if source
not in self
.knownBranches
:
2861 lostAndFoundBranches
.add(source
)
2863 # Perforce does not strictly require branches to be defined, so we also
2864 # check git config for a branch list.
2866 # Example of branch definition in git config file:
2868 # branchList=main:branchA
2869 # branchList=main:branchB
2870 # branchList=branchA:branchC
2871 configBranches
= gitConfigList("git-p4.branchList")
2872 for branch
in configBranches
:
2874 (source
, destination
) = branch
.split(":")
2875 self
.knownBranches
[destination
] = source
2877 lostAndFoundBranches
.discard(destination
)
2879 if source
not in self
.knownBranches
:
2880 lostAndFoundBranches
.add(source
)
2883 for branch
in lostAndFoundBranches
:
2884 self
.knownBranches
[branch
] = branch
2886 def getBranchMappingFromGitBranches(self
):
2887 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2888 for branch
in branches
.keys():
2889 if branch
== "master":
2892 branch
= branch
[len(self
.projectName
):]
2893 self
.knownBranches
[branch
] = branch
2895 def updateOptionDict(self
, d
):
2897 if self
.keepRepoPath
:
2898 option_keys
['keepRepoPath'] = 1
2900 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2902 def readOptions(self
, d
):
2903 self
.keepRepoPath
= (d
.has_key('options')
2904 and ('keepRepoPath' in d
['options']))
2906 def gitRefForBranch(self
, branch
):
2907 if branch
== "main":
2908 return self
.refPrefix
+ "master"
2910 if len(branch
) <= 0:
2913 return self
.refPrefix
+ self
.projectName
+ branch
2915 def gitCommitByP4Change(self
, ref
, change
):
2917 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2920 latestCommit
= parseRevision(ref
)
2924 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2925 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2930 log
= extractLogMessageFromGitCommit(next
)
2931 settings
= extractSettingsGitLog(log
)
2932 currentChange
= int(settings
['change'])
2934 print "current change %s" % currentChange
2936 if currentChange
== change
:
2938 print "found %s" % next
2941 if currentChange
< change
:
2942 earliestCommit
= "^%s" % next
2944 latestCommit
= "%s" % next
2948 def importNewBranch(self
, branch
, maxChange
):
2949 # make fast-import flush all changes to disk and update the refs using the checkpoint
2950 # command so that we can try to find the branch parent in the git history
2951 self
.gitStream
.write("checkpoint\n\n");
2952 self
.gitStream
.flush();
2953 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2954 range = "@1,%s" % maxChange
2955 #print "prefix" + branchPrefix
2956 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
2957 if len(changes
) <= 0:
2959 firstChange
= changes
[0]
2960 #print "first change in branch: %s" % firstChange
2961 sourceBranch
= self
.knownBranches
[branch
]
2962 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2963 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2964 #print "source " + sourceBranch
2966 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2967 #print "branch parent: %s" % branchParentChange
2968 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2969 if len(gitParent
) > 0:
2970 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2971 #print "parent git commit: %s" % gitParent
2973 self
.importChanges(changes
)
2976 def searchParent(self
, parent
, branch
, target
):
2978 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
2979 "--no-merges", parent
]):
2981 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2984 print "Found parent of %s in commit %s" % (branch
, blob
)
2991 def importChanges(self
, changes
):
2993 for change
in changes
:
2994 description
= p4_describe(change
)
2995 self
.updateOptionDict(description
)
2998 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3003 if self
.detectBranches
:
3004 branches
= self
.splitFilesIntoBranches(description
)
3005 for branch
in branches
.keys():
3007 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3008 self
.branchPrefixes
= [ branchPrefix
]
3012 filesForCommit
= branches
[branch
]
3015 print "branch is %s" % branch
3017 self
.updatedBranches
.add(branch
)
3019 if branch
not in self
.createdBranches
:
3020 self
.createdBranches
.add(branch
)
3021 parent
= self
.knownBranches
[branch
]
3022 if parent
== branch
:
3025 fullBranch
= self
.projectName
+ branch
3026 if fullBranch
not in self
.p4BranchesInGit
:
3028 print("\n Importing new branch %s" % fullBranch
);
3029 if self
.importNewBranch(branch
, change
- 1):
3031 self
.p4BranchesInGit
.append(fullBranch
)
3033 print("\n Resuming with change %s" % change
);
3036 print "parent determined through known branches: %s" % parent
3038 branch
= self
.gitRefForBranch(branch
)
3039 parent
= self
.gitRefForBranch(parent
)
3042 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
3044 if len(parent
) == 0 and branch
in self
.initialParents
:
3045 parent
= self
.initialParents
[branch
]
3046 del self
.initialParents
[branch
]
3050 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3052 print "Creating temporary branch: " + tempBranch
3053 self
.commit(description
, filesForCommit
, tempBranch
)
3054 self
.tempBranches
.append(tempBranch
)
3056 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3058 self
.commit(description
, filesForCommit
, branch
, blob
)
3061 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3062 self
.commit(description
, filesForCommit
, branch
, parent
)
3064 files
= self
.extractFilesFromCommit(description
)
3065 self
.commit(description
, files
, self
.branch
,
3067 # only needed once, to connect to the previous commit
3068 self
.initialParent
= ""
3070 print self
.gitError
.read()
3073 def importHeadRevision(self
, revision
):
3074 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3077 details
["user"] = "git perforce import user"
3078 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3079 % (' '.join(self
.depotPaths
), revision
))
3080 details
["change"] = revision
3084 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3086 for info
in p4CmdList(["files"] + fileArgs
):
3088 if 'code' in info
and info
['code'] == 'error':
3089 sys
.stderr
.write("p4 returned an error: %s\n"
3091 if info
['data'].find("must refer to client") >= 0:
3092 sys
.stderr
.write("This particular p4 error is misleading.\n")
3093 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3094 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3096 if 'p4ExitCode' in info
:
3097 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3101 change
= int(info
["change"])
3102 if change
> newestRevision
:
3103 newestRevision
= change
3105 if info
["action"] in self
.delete_actions
:
3106 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3107 #fileCnt = fileCnt + 1
3110 for prop
in ["depotFile", "rev", "action", "type" ]:
3111 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3113 fileCnt
= fileCnt
+ 1
3115 details
["change"] = newestRevision
3117 # Use time from top-most change so that all git p4 clones of
3118 # the same p4 repo have the same commit SHA1s.
3119 res
= p4_describe(newestRevision
)
3120 details
["time"] = res
["time"]
3122 self
.updateOptionDict(details
)
3124 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3126 print "IO error with git fast-import. Is your git version recent enough?"
3127 print self
.gitError
.read()
3130 def run(self
, args
):
3131 self
.depotPaths
= []
3132 self
.changeRange
= ""
3133 self
.previousDepotPaths
= []
3134 self
.hasOrigin
= False
3136 # map from branch depot path to parent branch
3137 self
.knownBranches
= {}
3138 self
.initialParents
= {}
3140 if self
.importIntoRemotes
:
3141 self
.refPrefix
= "refs/remotes/p4/"
3143 self
.refPrefix
= "refs/heads/p4/"
3145 if self
.syncWithOrigin
:
3146 self
.hasOrigin
= originP4BranchesExist()
3149 print 'Syncing with origin first, using "git fetch origin"'
3150 system("git fetch origin")
3152 branch_arg_given
= bool(self
.branch
)
3153 if len(self
.branch
) == 0:
3154 self
.branch
= self
.refPrefix
+ "master"
3155 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3156 system("git update-ref %s refs/heads/p4" % self
.branch
)
3157 system("git branch -D p4")
3159 # accept either the command-line option, or the configuration variable
3160 if self
.useClientSpec
:
3161 # will use this after clone to set the variable
3162 self
.useClientSpec_from_options
= True
3164 if gitConfigBool("git-p4.useclientspec"):
3165 self
.useClientSpec
= True
3166 if self
.useClientSpec
:
3167 self
.clientSpecDirs
= getClientSpec()
3169 # TODO: should always look at previous commits,
3170 # merge with previous imports, if possible.
3173 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3175 # branches holds mapping from branch name to sha1
3176 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3178 # restrict to just this one, disabling detect-branches
3179 if branch_arg_given
:
3180 short
= self
.branch
.split("/")[-1]
3181 if short
in branches
:
3182 self
.p4BranchesInGit
= [ short
]
3184 self
.p4BranchesInGit
= branches
.keys()
3186 if len(self
.p4BranchesInGit
) > 1:
3188 print "Importing from/into multiple branches"
3189 self
.detectBranches
= True
3190 for branch
in branches
.keys():
3191 self
.initialParents
[self
.refPrefix
+ branch
] = \
3195 print "branches: %s" % self
.p4BranchesInGit
3198 for branch
in self
.p4BranchesInGit
:
3199 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3201 settings
= extractSettingsGitLog(logMsg
)
3203 self
.readOptions(settings
)
3204 if (settings
.has_key('depot-paths')
3205 and settings
.has_key ('change')):
3206 change
= int(settings
['change']) + 1
3207 p4Change
= max(p4Change
, change
)
3209 depotPaths
= sorted(settings
['depot-paths'])
3210 if self
.previousDepotPaths
== []:
3211 self
.previousDepotPaths
= depotPaths
3214 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3215 prev_list
= prev
.split("/")
3216 cur_list
= cur
.split("/")
3217 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3218 if cur_list
[i
] <> prev_list
[i
]:
3222 paths
.append ("/".join(cur_list
[:i
+ 1]))
3224 self
.previousDepotPaths
= paths
3227 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3228 self
.changeRange
= "@%s,#head" % p4Change
3229 if not self
.silent
and not self
.detectBranches
:
3230 print "Performing incremental import into %s git branch" % self
.branch
3232 # accept multiple ref name abbreviations:
3233 # refs/foo/bar/branch -> use it exactly
3234 # p4/branch -> prepend refs/remotes/ or refs/heads/
3235 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3236 if not self
.branch
.startswith("refs/"):
3237 if self
.importIntoRemotes
:
3238 prepend
= "refs/remotes/"
3240 prepend
= "refs/heads/"
3241 if not self
.branch
.startswith("p4/"):
3243 self
.branch
= prepend
+ self
.branch
3245 if len(args
) == 0 and self
.depotPaths
:
3247 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3249 if self
.depotPaths
and self
.depotPaths
!= args
:
3250 print ("previous import used depot path %s and now %s was specified. "
3251 "This doesn't work!" % (' '.join (self
.depotPaths
),
3255 self
.depotPaths
= sorted(args
)
3260 # Make sure no revision specifiers are used when --changesfile
3262 bad_changesfile
= False
3263 if len(self
.changesFile
) > 0:
3264 for p
in self
.depotPaths
:
3265 if p
.find("@") >= 0 or p
.find("#") >= 0:
3266 bad_changesfile
= True
3269 die("Option --changesfile is incompatible with revision specifiers")
3272 for p
in self
.depotPaths
:
3273 if p
.find("@") != -1:
3274 atIdx
= p
.index("@")
3275 self
.changeRange
= p
[atIdx
:]
3276 if self
.changeRange
== "@all":
3277 self
.changeRange
= ""
3278 elif ',' not in self
.changeRange
:
3279 revision
= self
.changeRange
3280 self
.changeRange
= ""
3282 elif p
.find("#") != -1:
3283 hashIdx
= p
.index("#")
3284 revision
= p
[hashIdx
:]
3286 elif self
.previousDepotPaths
== []:
3287 # pay attention to changesfile, if given, else import
3288 # the entire p4 tree at the head revision
3289 if len(self
.changesFile
) == 0:
3292 p
= re
.sub ("\.\.\.$", "", p
)
3293 if not p
.endswith("/"):
3298 self
.depotPaths
= newPaths
3300 # --detect-branches may change this for each branch
3301 self
.branchPrefixes
= self
.depotPaths
3303 self
.loadUserMapFromCache()
3305 if self
.detectLabels
:
3308 if self
.detectBranches
:
3309 ## FIXME - what's a P4 projectName ?
3310 self
.projectName
= self
.guessProjectName()
3313 self
.getBranchMappingFromGitBranches()
3315 self
.getBranchMapping()
3317 print "p4-git branches: %s" % self
.p4BranchesInGit
3318 print "initial parents: %s" % self
.initialParents
3319 for b
in self
.p4BranchesInGit
:
3323 b
= b
[len(self
.projectName
):]
3324 self
.createdBranches
.add(b
)
3326 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
3328 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3329 stdin
=subprocess
.PIPE
,
3330 stdout
=subprocess
.PIPE
,
3331 stderr
=subprocess
.PIPE
);
3332 self
.gitOutput
= self
.importProcess
.stdout
3333 self
.gitStream
= self
.importProcess
.stdin
3334 self
.gitError
= self
.importProcess
.stderr
3337 self
.importHeadRevision(revision
)
3341 if len(self
.changesFile
) > 0:
3342 output
= open(self
.changesFile
).readlines()
3345 changeSet
.add(int(line
))
3347 for change
in changeSet
:
3348 changes
.append(change
)
3352 # catch "git p4 sync" with no new branches, in a repo that
3353 # does not have any existing p4 branches
3355 if not self
.p4BranchesInGit
:
3356 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3358 # The default branch is master, unless --branch is used to
3359 # specify something else. Make sure it exists, or complain
3360 # nicely about how to use --branch.
3361 if not self
.detectBranches
:
3362 if not branch_exists(self
.branch
):
3363 if branch_arg_given
:
3364 die("Error: branch %s does not exist." % self
.branch
)
3366 die("Error: no branch %s; perhaps specify one with --branch." %
3370 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3372 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3374 if len(self
.maxChanges
) > 0:
3375 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3377 if len(changes
) == 0:
3379 print "No changes to import!"
3381 if not self
.silent
and not self
.detectBranches
:
3382 print "Import destination: %s" % self
.branch
3384 self
.updatedBranches
= set()
3386 if not self
.detectBranches
:
3388 # start a new branch
3389 self
.initialParent
= ""
3391 # build on a previous revision
3392 self
.initialParent
= parseRevision(self
.branch
)
3394 self
.importChanges(changes
)
3398 if len(self
.updatedBranches
) > 0:
3399 sys
.stdout
.write("Updated branches: ")
3400 for b
in self
.updatedBranches
:
3401 sys
.stdout
.write("%s " % b
)
3402 sys
.stdout
.write("\n")
3404 if gitConfigBool("git-p4.importLabels"):
3405 self
.importLabels
= True
3407 if self
.importLabels
:
3408 p4Labels
= getP4Labels(self
.depotPaths
)
3409 gitTags
= getGitTags()
3411 missingP4Labels
= p4Labels
- gitTags
3412 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3414 self
.gitStream
.close()
3415 if self
.importProcess
.wait() != 0:
3416 die("fast-import failed: %s" % self
.gitError
.read())
3417 self
.gitOutput
.close()
3418 self
.gitError
.close()
3420 # Cleanup temporary branches created during import
3421 if self
.tempBranches
!= []:
3422 for branch
in self
.tempBranches
:
3423 read_pipe("git update-ref -d %s" % branch
)
3424 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3426 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3427 # a convenient shortcut refname "p4".
3428 if self
.importIntoRemotes
:
3429 head_ref
= self
.refPrefix
+ "HEAD"
3430 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3431 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3435 class P4Rebase(Command
):
3437 Command
.__init
__(self
)
3439 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3441 self
.importLabels
= False
3442 self
.description
= ("Fetches the latest revision from perforce and "
3443 + "rebases the current work (branch) against it")
3445 def run(self
, args
):
3447 sync
.importLabels
= self
.importLabels
3450 return self
.rebase()
3453 if os
.system("git update-index --refresh") != 0:
3454 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3455 if len(read_pipe("git diff-index HEAD --")) > 0:
3456 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3458 [upstream
, settings
] = findUpstreamBranchPoint()
3459 if len(upstream
) == 0:
3460 die("Cannot find upstream branchpoint for rebase")
3462 # the branchpoint may be p4/foo~3, so strip off the parent
3463 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3465 print "Rebasing the current branch onto %s" % upstream
3466 oldHead
= read_pipe("git rev-parse HEAD").strip()
3467 system("git rebase %s" % upstream
)
3468 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3471 class P4Clone(P4Sync
):
3473 P4Sync
.__init
__(self
)
3474 self
.description
= "Creates a new git repository and imports from Perforce into it"
3475 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3477 optparse
.make_option("--destination", dest
="cloneDestination",
3478 action
='store', default
=None,
3479 help="where to leave result of the clone"),
3480 optparse
.make_option("--bare", dest
="cloneBare",
3481 action
="store_true", default
=False),
3483 self
.cloneDestination
= None
3484 self
.needsGit
= False
3485 self
.cloneBare
= False
3487 def defaultDestination(self
, args
):
3488 ## TODO: use common prefix of args?
3490 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3491 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3492 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3493 depotDir
= re
.sub(r
"/$", "", depotDir
)
3494 return os
.path
.split(depotDir
)[1]
3496 def run(self
, args
):
3500 if self
.keepRepoPath
and not self
.cloneDestination
:
3501 sys
.stderr
.write("Must specify destination for --keep-path\n")
3506 if not self
.cloneDestination
and len(depotPaths
) > 1:
3507 self
.cloneDestination
= depotPaths
[-1]
3508 depotPaths
= depotPaths
[:-1]
3510 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3511 for p
in depotPaths
:
3512 if not p
.startswith("//"):
3513 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3516 if not self
.cloneDestination
:
3517 self
.cloneDestination
= self
.defaultDestination(args
)
3519 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3521 if not os
.path
.exists(self
.cloneDestination
):
3522 os
.makedirs(self
.cloneDestination
)
3523 chdir(self
.cloneDestination
)
3525 init_cmd
= [ "git", "init" ]
3527 init_cmd
.append("--bare")
3528 retcode
= subprocess
.call(init_cmd
)
3530 raise CalledProcessError(retcode
, init_cmd
)
3532 if not P4Sync
.run(self
, depotPaths
):
3535 # create a master branch and check out a work tree
3536 if gitBranchExists(self
.branch
):
3537 system([ "git", "branch", "master", self
.branch
])
3538 if not self
.cloneBare
:
3539 system([ "git", "checkout", "-f" ])
3541 print 'Not checking out any branch, use ' \
3542 '"git checkout -q -b master <branch>"'
3544 # auto-set this variable if invoked with --use-client-spec
3545 if self
.useClientSpec_from_options
:
3546 system("git config --bool git-p4.useclientspec true")
3550 class P4Branches(Command
):
3552 Command
.__init
__(self
)
3554 self
.description
= ("Shows the git branches that hold imports and their "
3555 + "corresponding perforce depot paths")
3556 self
.verbose
= False
3558 def run(self
, args
):
3559 if originP4BranchesExist():
3560 createOrUpdateBranchesFromOrigin()
3562 cmdline
= "git rev-parse --symbolic "
3563 cmdline
+= " --remotes"
3565 for line
in read_pipe_lines(cmdline
):
3568 if not line
.startswith('p4/') or line
== "p4/HEAD":
3572 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3573 settings
= extractSettingsGitLog(log
)
3575 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3578 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3580 optparse
.IndentedHelpFormatter
.__init
__(self
)
3582 def format_description(self
, description
):
3584 return description
+ "\n"
3588 def printUsage(commands
):
3589 print "usage: %s <command> [options]" % sys
.argv
[0]
3591 print "valid commands: %s" % ", ".join(commands
)
3593 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3598 "submit" : P4Submit
,
3599 "commit" : P4Submit
,
3601 "rebase" : P4Rebase
,
3603 "rollback" : P4RollBack
,
3604 "branches" : P4Branches
3609 if len(sys
.argv
[1:]) == 0:
3610 printUsage(commands
.keys())
3613 cmdName
= sys
.argv
[1]
3615 klass
= commands
[cmdName
]
3618 print "unknown command %s" % cmdName
3620 printUsage(commands
.keys())
3623 options
= cmd
.options
3624 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3628 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3630 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3632 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3634 description
= cmd
.description
,
3635 formatter
= HelpFormatter())
3637 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3639 verbose
= cmd
.verbose
3641 if cmd
.gitdir
== None:
3642 cmd
.gitdir
= os
.path
.abspath(".git")
3643 if not isValidGitDir(cmd
.gitdir
):
3644 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3645 if os
.path
.exists(cmd
.gitdir
):
3646 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3650 if not isValidGitDir(cmd
.gitdir
):
3651 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3652 cmd
.gitdir
+= "/.git"
3654 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3656 os
.environ
["GIT_DIR"] = cmd
.gitdir
3658 if not cmd
.run(args
):
3663 if __name__
== '__main__':